[ROCm][Perf] Add low-M FP32 router GEMM for gfx950 (#54845)
This commit is contained in:
@@ -1,16 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Platform-agnostic eligibility tests for GateLinear's fused router GEMM.
|
||||
"""Platform-agnostic eligibility tests for GateLinear's ROCm router GEMMs.
|
||||
|
||||
These assert the ``allow_cublas_router_gemm`` dispatch flag directly, so they
|
||||
run device-free by mocking the platform predicates. The flag decides whether
|
||||
the bf16xbf16->fp32 router GEMM uses ``torch.mm``'s fused out_dtype epilogue
|
||||
(one kernel) or falls back to a bf16 matmul plus a standalone bf16->fp32 copy.
|
||||
These assert the dispatch flags directly, so they run device-free by mocking
|
||||
the platform predicates. ``allow_cublas_router_gemm`` selects the
|
||||
bf16xbf16->fp32 ``torch.mm`` epilogue, while ``allow_fp32_router_gemm`` selects
|
||||
the gfx950 low-M kernel with fp32 weights and output.
|
||||
|
||||
The ROCm branch is guarded on ``not bias`` because ``torch.mm`` has no bias
|
||||
term; a biased gate must fall back so the bias is not silently dropped.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.router.gate_linear as gate_linear_mod
|
||||
@@ -25,6 +26,10 @@ def _make_gate(
|
||||
bias: bool = False,
|
||||
params_dtype: torch.dtype = torch.bfloat16,
|
||||
out_dtype: torch.dtype | None = torch.float32,
|
||||
input_size: int = 2048,
|
||||
output_size: int = 64,
|
||||
on_gfx950: bool = False,
|
||||
parallel_world_size: int = 1,
|
||||
) -> GateLinear:
|
||||
"""Build a GateLinear with platform predicates mocked, no GPU needed."""
|
||||
for target in (
|
||||
@@ -37,20 +42,22 @@ def _make_gate(
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
f"{target}.get_tensor_model_parallel_world_size",
|
||||
lambda: 1,
|
||||
lambda: parallel_world_size,
|
||||
)
|
||||
|
||||
platform = gate_linear_mod.current_platform
|
||||
monkeypatch.setattr(platform, "is_cuda", lambda: is_cuda)
|
||||
monkeypatch.setattr(platform, "is_rocm", lambda: is_rocm)
|
||||
# Force the CUDA specialized-kernel gate off so ROCm eligibility is the
|
||||
# only thing under test (these are the SM90/SM100 capability checks).
|
||||
monkeypatch.setattr(platform, "is_device_capability", lambda *a, **k: False)
|
||||
monkeypatch.setattr(platform, "is_device_capability_family", lambda *a, **k: False)
|
||||
if is_rocm:
|
||||
import vllm.platforms.rocm as rocm_platform
|
||||
|
||||
monkeypatch.setattr(rocm_platform, "on_gfx950", lambda: on_gfx950)
|
||||
|
||||
return GateLinear(
|
||||
input_size=2048,
|
||||
output_size=64,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
bias=bias,
|
||||
out_dtype=out_dtype,
|
||||
params_dtype=params_dtype,
|
||||
@@ -96,3 +103,68 @@ def test_rocm_set_out_dtype_respects_bias_guard(monkeypatch):
|
||||
gate = _make_gate(monkeypatch, is_rocm=True, bias=True, out_dtype=None)
|
||||
gate.set_out_dtype(torch.float32)
|
||||
assert not gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("input_size", "output_size"),
|
||||
[(3072, 256), (4096, 8), (4096, 192), (6144, 128), (6144, 256)],
|
||||
)
|
||||
def test_rocm_gfx950_enables_fp32_router_gemm(
|
||||
monkeypatch, input_size: int, output_size: int
|
||||
) -> None:
|
||||
gate = _make_gate(
|
||||
monkeypatch,
|
||||
is_rocm=True,
|
||||
params_dtype=torch.float32,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
on_gfx950=True,
|
||||
)
|
||||
assert gate.allow_fp32_router_gemm
|
||||
assert not gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ep_size", [2, 4, 8])
|
||||
def test_rocm_fp32_router_gemm_is_replicated_across_ep_sizes(
|
||||
monkeypatch, ep_size: int
|
||||
) -> None:
|
||||
gate = _make_gate(
|
||||
monkeypatch,
|
||||
is_rocm=True,
|
||||
params_dtype=torch.float32,
|
||||
input_size=6144,
|
||||
output_size=128,
|
||||
on_gfx950=True,
|
||||
parallel_world_size=ep_size,
|
||||
)
|
||||
assert gate.weight.shape == (128, 6144)
|
||||
assert gate.allow_fp32_router_gemm
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("input_size", "output_size", "params_dtype", "bias", "on_gfx950"),
|
||||
[
|
||||
pytest.param(6144, 128, torch.float32, False, False, id="gfx942"),
|
||||
pytest.param(2048, 64, torch.float32, False, True, id="shape"),
|
||||
pytest.param(6144, 128, torch.bfloat16, False, True, id="bf16-weight"),
|
||||
pytest.param(6144, 128, torch.float32, True, True, id="bias"),
|
||||
],
|
||||
)
|
||||
def test_rocm_fp32_router_gemm_rejects_unsupported_configs(
|
||||
monkeypatch,
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
bias: bool,
|
||||
on_gfx950: bool,
|
||||
) -> None:
|
||||
gate = _make_gate(
|
||||
monkeypatch,
|
||||
is_rocm=True,
|
||||
params_dtype=params_dtype,
|
||||
bias=bias,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
on_gfx950=on_gfx950,
|
||||
)
|
||||
assert not gate.allow_fp32_router_gemm
|
||||
|
||||
Reference in New Issue
Block a user