[ROCm][Perf] Use a stacked hipBLASLt GEMM for the bf16x3 router (#52668)
Signed-off-by: Aakif Nawaz <aakif.nawaz@amd.com>
This commit is contained in:
@@ -11,6 +11,8 @@ The ROCm branch is guarded on ``not bias`` because ``torch.mm`` has no bias
|
||||
term; a biased gate must fall back so the bias is not silently dropped.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -168,3 +170,91 @@ def test_rocm_fp32_router_gemm_rejects_unsupported_configs(
|
||||
on_gfx950=on_gfx950,
|
||||
)
|
||||
assert not gate.allow_fp32_router_gemm
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ROCm BF16x3 tier (fp32 router weights)
|
||||
# ---------------------------------------------------------------------------
|
||||
def _make_bf16x3_gate(monkeypatch, *, supported=True, **kwargs):
|
||||
"""Build a gate with the ROCm bf16x3 platform check mocked."""
|
||||
from vllm.model_executor.layers.fused_moe.router import (
|
||||
bf16x3_router_gemm_rocm as rocm_bf16x3,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rocm_bf16x3, "platform_supported", lambda: supported)
|
||||
kwargs.setdefault("params_dtype", torch.float32)
|
||||
return _make_gate(monkeypatch, is_rocm=True, **kwargs)
|
||||
|
||||
|
||||
def test_rocm_bf16x3_enabled_for_fp32_weight(monkeypatch):
|
||||
gate = _make_bf16x3_gate(monkeypatch)
|
||||
assert gate.allow_rocm_bf16x3_router_gemm
|
||||
# Built by process_weights_after_loading, not lazily in forward().
|
||||
# Registered up front but still None, so named_buffers() skips it.
|
||||
assert gate._bf16x3_weight is None
|
||||
assert "_bf16x3_weight" in gate._buffers
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_by_bias(monkeypatch):
|
||||
# torch.mm has no bias term.
|
||||
gate = _make_bf16x3_gate(monkeypatch, bias=True)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_for_bf16_weight(monkeypatch):
|
||||
# A bf16 weight belongs to the cuBLAS tier; there is nothing to split.
|
||||
gate = _make_bf16x3_gate(monkeypatch, params_dtype=torch.bfloat16)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_for_non_fp32_out_dtype(monkeypatch):
|
||||
gate = _make_bf16x3_gate(monkeypatch, out_dtype=torch.bfloat16)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_on_unsupported_arch(monkeypatch):
|
||||
# Only gfx950 is benchmarked; gfx942 has a different fp32:bf16 MFMA ratio.
|
||||
gate = _make_bf16x3_gate(monkeypatch, supported=False)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
assert not gate._rocm_bf16x3_weight_eligible
|
||||
|
||||
|
||||
def test_rocm_bf16x3_no_split_built_when_tier_disabled(monkeypatch):
|
||||
"""An ineligible out_dtype must not allocate a split that is never read;
|
||||
it is 1.5x the router weight on every layer."""
|
||||
gate = _make_bf16x3_gate(monkeypatch, out_dtype=torch.bfloat16)
|
||||
gate.quant_method.process_weights_after_loading(gate)
|
||||
assert gate._bf16x3_weight is None
|
||||
|
||||
|
||||
def test_rocm_bf16x3_set_out_dtype_enables_tier(monkeypatch):
|
||||
gate = _make_bf16x3_gate(monkeypatch, out_dtype=None)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
gate.set_out_dtype(torch.float32)
|
||||
assert gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_dispatch_is_a_custom_op():
|
||||
"""The num_tokens branch must stay opaque to Dynamo; see the op's impl."""
|
||||
assert hasattr(torch.ops.vllm, "rocm_bf16x3_router_gemm_dispatch")
|
||||
x = torch.zeros(8192, 6144, dtype=torch.bfloat16, device="meta")
|
||||
w = torch.zeros(128, 6144, dtype=torch.float32, device="meta")
|
||||
split = torch.zeros(3, 128, 6144, dtype=torch.bfloat16, device="meta")
|
||||
out = torch.ops.vllm.rocm_bf16x3_router_gemm_dispatch(x, w, split)
|
||||
assert out.shape == (8192, 128)
|
||||
assert out.dtype == torch.float32
|
||||
|
||||
|
||||
def test_rocm_bf16x3_is_reachable_for_low_m_shapes(monkeypatch):
|
||||
"""The gfx950 low-M fp32 tier claims these shapes too and returns
|
||||
unconditionally, so bf16x3 is only reachable if checked first."""
|
||||
gate = _make_bf16x3_gate(
|
||||
monkeypatch, input_size=6144, output_size=128, on_gfx950=True
|
||||
)
|
||||
assert gate.allow_fp32_router_gemm
|
||||
assert gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
source = inspect.getsource(type(gate).forward)
|
||||
assert source.index("allow_rocm_bf16x3_router_gemm") < source.index(
|
||||
"allow_fp32_router_gemm"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user