[ROCm][Perf] Use a stacked hipBLASLt GEMM for the bf16x3 router (#52668)
Signed-off-by: Aakif Nawaz <aakif.nawaz@amd.com>
This commit is contained in:
@@ -0,0 +1,201 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the ROCm BF16x3 router GEMM.
|
||||
|
||||
The weight split and the eligibility policy are pure functions, so most of
|
||||
this runs device-free. Only the GEMM correctness tests need a gfx950 runner.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.router import (
|
||||
bf16x3_router_gemm_rocm as rocm_bf16x3,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
|
||||
# (hidden_size, num_experts) for the models that ship fp32 router weights.
|
||||
# (6144, 128) -> MiniMax-M3, (3072, 256) -> MiniMax-M2/M2.5
|
||||
ROUTER_SHAPES = [(6144, 128), (3072, 256)]
|
||||
ROUTER_WEIGHT_SCALE = 0.053
|
||||
|
||||
|
||||
def _requires_gfx950():
|
||||
if not rocm_bf16x3.platform_supported():
|
||||
pytest.skip("ROCm bf16x3 router GEMM requires gfx950")
|
||||
|
||||
|
||||
def _inputs(num_tokens, hidden_size, num_experts, seed=42, device="cuda"):
|
||||
torch.manual_seed(seed)
|
||||
x = torch.randn(num_tokens, hidden_size, device=device).to(torch.bfloat16)
|
||||
w = torch.randn(num_experts, hidden_size, dtype=torch.float32, device=device)
|
||||
return x, w * ROUTER_WEIGHT_SCALE
|
||||
|
||||
|
||||
def _top_set(logits, k=4):
|
||||
"""Sorted top-k indices; topk's order among near-ties is not meaningful."""
|
||||
return logits.topk(k, dim=-1).indices.sort(dim=-1).values
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("hidden_size", "num_experts"), ROUTER_SHAPES)
|
||||
def test_split_reconstructs_exactly(hidden_size: int, num_experts: int):
|
||||
torch.manual_seed(42)
|
||||
w = torch.randn(num_experts, hidden_size) * ROUTER_WEIGHT_SCALE
|
||||
split = rocm_bf16x3.split_bf16x3(w)
|
||||
|
||||
assert split.shape == (rocm_bf16x3.BF16X3_TERMS, num_experts, hidden_size)
|
||||
assert split.dtype == torch.bfloat16
|
||||
# router_gemm reinterprets this as (TERMS * E, K) without a copy.
|
||||
assert split.is_contiguous()
|
||||
assert torch.equal(split.float().sum(0), w)
|
||||
|
||||
|
||||
def test_split_is_exact_across_normal_exponents():
|
||||
exps = torch.arange(-100, 100, dtype=torch.float32)
|
||||
mantissas = torch.linspace(1.0, 1.999, 64, dtype=torch.float32)
|
||||
w = (mantissas[:, None] * torch.pow(2.0, exps)[None, :]).contiguous()
|
||||
w = torch.cat([w, -w], dim=0)
|
||||
assert torch.equal(rocm_bf16x3.split_bf16x3(w).float().sum(0), w)
|
||||
|
||||
|
||||
def test_split_rejects_non_finite():
|
||||
"""ValueError rather than assert: the caller downgrades to the fp32 path
|
||||
instead of failing model load, and this must survive ``python -O``."""
|
||||
for bad in (float("inf"), float("-inf"), float("nan")):
|
||||
with pytest.raises(ValueError):
|
||||
rocm_bf16x3.split_bf16x3(torch.full((2, 4), bad, dtype=torch.float32))
|
||||
|
||||
|
||||
def test_split_rejects_non_fp32():
|
||||
with pytest.raises(ValueError):
|
||||
rocm_bf16x3.split_bf16x3(torch.zeros(2, 4, dtype=torch.bfloat16))
|
||||
|
||||
|
||||
def test_split_does_not_mutate_input():
|
||||
w = torch.randn(8, 32, dtype=torch.float32)
|
||||
original = w.clone()
|
||||
rocm_bf16x3.split_bf16x3(w)
|
||||
assert torch.equal(w, original)
|
||||
|
||||
|
||||
def test_is_supported_rejects_ineligible_inputs():
|
||||
meta = torch.device("meta")
|
||||
x = torch.zeros(8192, 6144, dtype=torch.bfloat16, device=meta)
|
||||
w = torch.zeros(128, 6144, dtype=torch.float32, device=meta)
|
||||
assert rocm_bf16x3.is_supported(x, w)
|
||||
|
||||
assert not rocm_bf16x3.is_supported(x[: rocm_bf16x3.MIN_TOKENS - 1], w)
|
||||
assert not rocm_bf16x3.is_supported(x.float(), w)
|
||||
assert not rocm_bf16x3.is_supported(x, w.bfloat16())
|
||||
assert not rocm_bf16x3.is_supported(x, w[:, :128])
|
||||
assert not rocm_bf16x3.is_supported(x.T.contiguous().T, w)
|
||||
|
||||
|
||||
def test_is_supported_rejects_mismatched_devices():
|
||||
"""The weight can be offloaded while the activations are not."""
|
||||
x = torch.zeros(8192, 6144, dtype=torch.bfloat16, device="meta")
|
||||
w = torch.zeros(128, 6144, dtype=torch.float32)
|
||||
assert not rocm_bf16x3.is_supported(x, w)
|
||||
|
||||
|
||||
def test_is_supported_accepts_odd_expert_counts():
|
||||
"""Unlike a tl.arange-indexed kernel, torch.mm needs no power-of-two E."""
|
||||
meta = torch.device("meta")
|
||||
x = torch.zeros(8192, 6144, dtype=torch.bfloat16, device=meta)
|
||||
for num_experts in (17, 384, 512):
|
||||
w = torch.zeros(num_experts, 6144, dtype=torch.float32, device=meta)
|
||||
assert rocm_bf16x3.is_supported(x, w)
|
||||
|
||||
|
||||
# Measured on gfx950: this path lands at 3.4e-7 to 4.9e-7 and varies <0.2%
|
||||
# across token counts. Fixed bounds rather than a live comparison, because the
|
||||
# fp32 fallback swings 5.7e-7 to 1.4e-6 as torch picks different hipBLASLt
|
||||
# kernels at different M. _FP32_FALLBACK_BEST is the best the fallback was ever
|
||||
# observed to do, so beating it proves the claim without racing a moving target.
|
||||
_MAX_REL_L2 = 2e-6
|
||||
_FP32_FALLBACK_BEST = 5.7e-7
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("hidden_size", "num_experts"), ROUTER_SHAPES)
|
||||
@pytest.mark.parametrize("num_tokens", [2048, 8192, 16385])
|
||||
def test_matches_fp64_reference(hidden_size: int, num_experts: int, num_tokens: int):
|
||||
_requires_gfx950()
|
||||
x, w = _inputs(num_tokens, hidden_size, num_experts)
|
||||
out = rocm_bf16x3.bf16x3_router_gemm(x, rocm_bf16x3.split_bf16x3(w))
|
||||
ref = x.double() @ w.double().T
|
||||
|
||||
assert out.shape == (num_tokens, num_experts)
|
||||
assert out.dtype == torch.float32
|
||||
assert (out.double() - ref).norm() / ref.norm() < _MAX_REL_L2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_experts", [17, 384])
|
||||
def test_matches_fp64_for_non_power_of_two_experts(num_experts: int):
|
||||
"""Exercises the reduction's N mask, which the model shapes never hit."""
|
||||
_requires_gfx950()
|
||||
x, w = _inputs(8192, 6144, num_experts)
|
||||
out = rocm_bf16x3.bf16x3_router_gemm(x, rocm_bf16x3.split_bf16x3(w))
|
||||
ref = x.double() @ w.double().T
|
||||
|
||||
assert out.shape == (8192, num_experts)
|
||||
assert (out.double() - ref).norm() / ref.norm() < _MAX_REL_L2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("hidden_size", "num_experts"), ROUTER_SHAPES)
|
||||
def test_more_accurate_than_fp32_fallback(hidden_size: int, num_experts: int):
|
||||
"""The claim is not just accuracy but beating the fp32 GEMM it replaces."""
|
||||
_requires_gfx950()
|
||||
x, w = _inputs(16384, hidden_size, num_experts)
|
||||
ref = x.double() @ w.double().T
|
||||
out = rocm_bf16x3.bf16x3_router_gemm(x, rocm_bf16x3.split_bf16x3(w))
|
||||
assert (out.double() - ref).norm() / ref.norm() < _FP32_FALLBACK_BEST
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("hidden_size", "num_experts"), ROUTER_SHAPES)
|
||||
def test_expert_selection_matches_fp64(hidden_size: int, num_experts: int):
|
||||
"""Relative L2 is a proxy; the expert set is what reaches the model.
|
||||
|
||||
Measures zero misroutes on both shapes, where the fp32 fallback misroutes a
|
||||
nonzero number at some token counts.
|
||||
"""
|
||||
_requires_gfx950()
|
||||
num_tokens = 16384
|
||||
x, w = _inputs(num_tokens, hidden_size, num_experts)
|
||||
ref_set = _top_set(x.double() @ w.double().T)
|
||||
out = rocm_bf16x3.bf16x3_router_gemm(x, rocm_bf16x3.split_bf16x3(w))
|
||||
|
||||
misrouted = int((_top_set(out) != ref_set).any(-1).sum())
|
||||
assert misrouted / num_tokens < 1e-4
|
||||
|
||||
|
||||
def test_writes_into_supplied_out():
|
||||
_requires_gfx950()
|
||||
x, w = _inputs(8192, 6144, 128)
|
||||
split = rocm_bf16x3.split_bf16x3(w)
|
||||
out = torch.empty(8192, 128, dtype=torch.float32, device="cuda")
|
||||
|
||||
returned = rocm_bf16x3.bf16x3_router_gemm(x, split, out=out)
|
||||
assert returned is out
|
||||
assert torch.equal(out, rocm_bf16x3.bf16x3_router_gemm(x, split))
|
||||
|
||||
|
||||
def test_validates_layout_assumptions():
|
||||
"""The reduction assumes a contiguous (TERMS, E, K) split and a contiguous
|
||||
out; nothing in the kernel enforces either."""
|
||||
_requires_gfx950()
|
||||
x, w = _inputs(8192, 6144, 128)
|
||||
split = rocm_bf16x3.split_bf16x3(w)
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
rocm_bf16x3.bf16x3_router_gemm(x, split.transpose(1, 2))
|
||||
with pytest.raises(AssertionError):
|
||||
rocm_bf16x3.bf16x3_router_gemm(x.float(), split)
|
||||
with pytest.raises(AssertionError):
|
||||
rocm_bf16x3.bf16x3_router_gemm(x, split[:2])
|
||||
with pytest.raises(AssertionError):
|
||||
rocm_bf16x3.bf16x3_router_gemm(x[:, :512], split)
|
||||
wide = torch.empty(8192, 256, dtype=torch.float32, device="cuda")
|
||||
with pytest.raises(AssertionError):
|
||||
rocm_bf16x3.bf16x3_router_gemm(x, split, out=wide[:, :128])
|
||||
@@ -11,6 +11,8 @@ The ROCm branch is guarded on ``not bias`` because ``torch.mm`` has no bias
|
||||
term; a biased gate must fall back so the bias is not silently dropped.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -168,3 +170,91 @@ def test_rocm_fp32_router_gemm_rejects_unsupported_configs(
|
||||
on_gfx950=on_gfx950,
|
||||
)
|
||||
assert not gate.allow_fp32_router_gemm
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ROCm BF16x3 tier (fp32 router weights)
|
||||
# ---------------------------------------------------------------------------
|
||||
def _make_bf16x3_gate(monkeypatch, *, supported=True, **kwargs):
|
||||
"""Build a gate with the ROCm bf16x3 platform check mocked."""
|
||||
from vllm.model_executor.layers.fused_moe.router import (
|
||||
bf16x3_router_gemm_rocm as rocm_bf16x3,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rocm_bf16x3, "platform_supported", lambda: supported)
|
||||
kwargs.setdefault("params_dtype", torch.float32)
|
||||
return _make_gate(monkeypatch, is_rocm=True, **kwargs)
|
||||
|
||||
|
||||
def test_rocm_bf16x3_enabled_for_fp32_weight(monkeypatch):
|
||||
gate = _make_bf16x3_gate(monkeypatch)
|
||||
assert gate.allow_rocm_bf16x3_router_gemm
|
||||
# Built by process_weights_after_loading, not lazily in forward().
|
||||
# Registered up front but still None, so named_buffers() skips it.
|
||||
assert gate._bf16x3_weight is None
|
||||
assert "_bf16x3_weight" in gate._buffers
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_by_bias(monkeypatch):
|
||||
# torch.mm has no bias term.
|
||||
gate = _make_bf16x3_gate(monkeypatch, bias=True)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_for_bf16_weight(monkeypatch):
|
||||
# A bf16 weight belongs to the cuBLAS tier; there is nothing to split.
|
||||
gate = _make_bf16x3_gate(monkeypatch, params_dtype=torch.bfloat16)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_for_non_fp32_out_dtype(monkeypatch):
|
||||
gate = _make_bf16x3_gate(monkeypatch, out_dtype=torch.bfloat16)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_on_unsupported_arch(monkeypatch):
|
||||
# Only gfx950 is benchmarked; gfx942 has a different fp32:bf16 MFMA ratio.
|
||||
gate = _make_bf16x3_gate(monkeypatch, supported=False)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
assert not gate._rocm_bf16x3_weight_eligible
|
||||
|
||||
|
||||
def test_rocm_bf16x3_no_split_built_when_tier_disabled(monkeypatch):
|
||||
"""An ineligible out_dtype must not allocate a split that is never read;
|
||||
it is 1.5x the router weight on every layer."""
|
||||
gate = _make_bf16x3_gate(monkeypatch, out_dtype=torch.bfloat16)
|
||||
gate.quant_method.process_weights_after_loading(gate)
|
||||
assert gate._bf16x3_weight is None
|
||||
|
||||
|
||||
def test_rocm_bf16x3_set_out_dtype_enables_tier(monkeypatch):
|
||||
gate = _make_bf16x3_gate(monkeypatch, out_dtype=None)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
gate.set_out_dtype(torch.float32)
|
||||
assert gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_dispatch_is_a_custom_op():
|
||||
"""The num_tokens branch must stay opaque to Dynamo; see the op's impl."""
|
||||
assert hasattr(torch.ops.vllm, "rocm_bf16x3_router_gemm_dispatch")
|
||||
x = torch.zeros(8192, 6144, dtype=torch.bfloat16, device="meta")
|
||||
w = torch.zeros(128, 6144, dtype=torch.float32, device="meta")
|
||||
split = torch.zeros(3, 128, 6144, dtype=torch.bfloat16, device="meta")
|
||||
out = torch.ops.vllm.rocm_bf16x3_router_gemm_dispatch(x, w, split)
|
||||
assert out.shape == (8192, 128)
|
||||
assert out.dtype == torch.float32
|
||||
|
||||
|
||||
def test_rocm_bf16x3_is_reachable_for_low_m_shapes(monkeypatch):
|
||||
"""The gfx950 low-M fp32 tier claims these shapes too and returns
|
||||
unconditionally, so bf16x3 is only reachable if checked first."""
|
||||
gate = _make_bf16x3_gate(
|
||||
monkeypatch, input_size=6144, output_size=128, on_gfx950=True
|
||||
)
|
||||
assert gate.allow_fp32_router_gemm
|
||||
assert gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
source = inspect.getsource(type(gate).forward)
|
||||
assert source.index("allow_rocm_bf16x3_router_gemm") < source.index(
|
||||
"allow_fp32_router_gemm"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user