[ROCm][Perf] Any-shape low-latency MoE router GEMM on MI355 (#61895)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
642c18d64c
commit
b466fbacd2
@@ -24,14 +24,15 @@ class GateLinear(ReplicatedLinear):
|
||||
|
||||
1. cuteDSL ll_bf16_gemm (SM90+, M<=16, bf16 in, fp32 out,
|
||||
any shape w/ K divisible by 8)
|
||||
2. fp32 specialized kernel (SM90+ or gfx950, bf16/fp32 in, fp32 out,
|
||||
M<=32, model-specific shapes)
|
||||
2. fp32 specialized kernel (bf16/fp32 in, fp32 out, M<=32; SM90+
|
||||
model-specific shapes, gfx950 any shape)
|
||||
3. cuteDSL ll_fp32w_gemm (SM90+, M<=32, bf16/fp32 in, fp32 out,
|
||||
any shape)
|
||||
4. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
|
||||
5. ROCm bf16x3 router GEMM (gfx950, bf16 in, fp32 weight, fp32 out,
|
||||
M>=2048)
|
||||
6. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
|
||||
6. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype); gfx950
|
||||
routes low M through the tier 2 kernel
|
||||
7. F.linear via ReplicatedLinear (ultimate fallback)
|
||||
|
||||
The ``out_dtype`` attribute is mutable and can be set after init
|
||||
@@ -73,16 +74,6 @@ class GateLinear(ReplicatedLinear):
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
is_gfx950 = on_gfx950()
|
||||
is_rocm_fp32_shape = False
|
||||
if is_gfx950:
|
||||
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
|
||||
ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES,
|
||||
)
|
||||
|
||||
is_rocm_fp32_shape = (
|
||||
input_size,
|
||||
output_size,
|
||||
) in ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES
|
||||
batch_invariant = envs.VLLM_BATCH_INVARIANT
|
||||
can_use_specialized_kernels = (
|
||||
current_platform.is_cuda()
|
||||
@@ -107,6 +98,7 @@ class GateLinear(ReplicatedLinear):
|
||||
return_bias=return_bias,
|
||||
)
|
||||
self.out_dtype = out_dtype
|
||||
self.is_gfx950 = is_gfx950
|
||||
|
||||
# A quantized gate exposes no plain ``weight``, so every specialized
|
||||
# tier below is disabled and forward falls back to ReplicatedLinear.
|
||||
@@ -127,7 +119,7 @@ class GateLinear(ReplicatedLinear):
|
||||
and (is_hopper or is_blackwell)
|
||||
and (input_size, output_size) in self.FP32_SUPPORTED_SHAPES
|
||||
)
|
||||
or (is_gfx950 and is_rocm_fp32_shape)
|
||||
or is_gfx950
|
||||
)
|
||||
)
|
||||
self.allow_bf16x3_router_gemm = (
|
||||
@@ -349,7 +341,10 @@ class GateLinear(ReplicatedLinear):
|
||||
|
||||
# Tier 6: cuBLAS bf16→fp32
|
||||
if self.allow_cublas_router_gemm and x.dtype == torch.bfloat16:
|
||||
output = torch.mm(x, self.weight.T, out_dtype=torch.float32)
|
||||
if self.is_gfx950:
|
||||
output = torch.ops.vllm.fp32_router_gemm_dispatch(x, self.weight, False)
|
||||
else:
|
||||
output = torch.mm(x, self.weight.T, out_dtype=torch.float32)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 7: F.linear (ReplicatedLinear)
|
||||
@@ -465,17 +460,20 @@ def fp32_router_gemm_dispatch_impl(
|
||||
This must be wrapped in a custom op because our torch.compile integration
|
||||
does not support runtime dispatching on num_tokens.
|
||||
"""
|
||||
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
|
||||
if current_platform.is_rocm():
|
||||
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
|
||||
can_use_rocm_fp32_router_gemm,
|
||||
rocm_fp32_router_gemm,
|
||||
)
|
||||
if current_platform.is_rocm():
|
||||
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
|
||||
can_use_rocm_fp32_router_gemm,
|
||||
rocm_fp32_router_gemm,
|
||||
)
|
||||
|
||||
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
|
||||
x = x.contiguous()
|
||||
if can_use_rocm_fp32_router_gemm(x, weight):
|
||||
return rocm_fp32_router_gemm(x, weight)
|
||||
return torch.nn.functional.linear(x.float(), weight)
|
||||
if weight.dtype == torch.bfloat16:
|
||||
return torch.mm(x, weight.T, out_dtype=torch.float32)
|
||||
return torch.nn.functional.linear(x.float(), weight)
|
||||
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
|
||||
return ops.fp32_router_gemm(x, weight)
|
||||
|
||||
if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Low-token FP32 router GEMM for gfx950."""
|
||||
"""Low-token router GEMM with FP32 output for gfx950 (FP32 or BF16 weights)."""
|
||||
|
||||
import torch
|
||||
|
||||
@@ -9,15 +9,10 @@ from vllm.platforms.rocm import on_gfx950
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
_MAX_TOKENS = 32
|
||||
ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES = frozenset(
|
||||
{
|
||||
(3072, 256),
|
||||
(4096, 8),
|
||||
(4096, 192),
|
||||
(6144, 128),
|
||||
(6144, 256),
|
||||
}
|
||||
)
|
||||
# From this many multiply-adds on, F.linear wins on MI355.
|
||||
_MAX_MACS = 1 << 27
|
||||
# Above this num_tokens * num_experts, hipBLASLt wins for BF16 weights.
|
||||
_MAX_BF16_TOKEN_EXPERTS = 8192
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -30,6 +25,7 @@ def _rocm_fp32_router_gemm_kernel(
|
||||
N: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
pid_m = pid // N
|
||||
@@ -40,12 +36,27 @@ def _rocm_fp32_router_gemm_kernel(
|
||||
|
||||
for k_start in range(0, K, BLOCK_K):
|
||||
offsets_k_block = k_start + offsets_k
|
||||
hidden_states = tl.load(
|
||||
hidden_states_ptr + offsets_m[:, None] * K + offsets_k_block[None, :],
|
||||
mask=offsets_m[:, None] < M,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
router_weight = tl.load(router_weight_ptr + pid_n * K + offsets_k_block)
|
||||
if EVEN_K:
|
||||
hidden_states = tl.load(
|
||||
hidden_states_ptr + offsets_m[:, None] * K + offsets_k_block[None, :],
|
||||
mask=offsets_m[:, None] < M,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
router_weight = tl.load(router_weight_ptr + pid_n * K + offsets_k_block).to(
|
||||
tl.float32
|
||||
)
|
||||
else:
|
||||
k_mask = offsets_k_block < K
|
||||
hidden_states = tl.load(
|
||||
hidden_states_ptr + offsets_m[:, None] * K + offsets_k_block[None, :],
|
||||
mask=(offsets_m[:, None] < M) & k_mask[None, :],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
router_weight = tl.load(
|
||||
router_weight_ptr + pid_n * K + offsets_k_block,
|
||||
mask=k_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
partials += hidden_states * router_weight[None, :]
|
||||
|
||||
accumulator = tl.sum(partials, axis=1)
|
||||
@@ -57,28 +68,36 @@ def _rocm_fp32_router_gemm_kernel(
|
||||
|
||||
|
||||
def _launch_config(
|
||||
hidden_size: int,
|
||||
num_experts: int,
|
||||
num_tokens: int,
|
||||
num_experts: int, num_tokens: int, bf16_weight: bool
|
||||
) -> tuple[int, int, int]:
|
||||
if (hidden_size, num_experts) == (4096, 8):
|
||||
return 1, 1024, 4
|
||||
"""Return (BLOCK_M, BLOCK_K, num_warps), swept on MI355 for M<=32.
|
||||
|
||||
One program computes BLOCK_M tokens for one expert, so narrow routers keep
|
||||
BLOCK_M=1 to fill the GPU and wide routers grow BLOCK_M to reuse weights.
|
||||
"""
|
||||
if bf16_weight:
|
||||
return (1 if num_tokens <= 4 else 2), 1024, 2
|
||||
if num_experts < 64:
|
||||
return 1, 1024, 4
|
||||
wide = num_experts > 256
|
||||
if num_tokens <= 2:
|
||||
return 1, 512, 2
|
||||
if num_tokens <= 4:
|
||||
return 1, 1024, 8
|
||||
return (2, 1024, 2) if wide else (2, 2048, 4)
|
||||
if num_tokens <= 8:
|
||||
return (4, 512, 2) if wide else (4, 1024, 4)
|
||||
if num_tokens <= 16:
|
||||
block_m = 8 if (hidden_size, num_experts) == (6144, 256) else 4
|
||||
return block_m, 1024, 8
|
||||
if (hidden_size, num_experts) == (3072, 256):
|
||||
return 8, 1024, 8
|
||||
return 4, 2048, 8
|
||||
return (8, 512, 2) if wide else (4, 1024, 4)
|
||||
if num_experts >= 768:
|
||||
return 16, 256, 2
|
||||
return (8, 256, 2) if wide else (4, 512, 2)
|
||||
|
||||
|
||||
def can_use_rocm_fp32_router_gemm(
|
||||
hidden_states: torch.Tensor,
|
||||
router_weight: torch.Tensor,
|
||||
) -> bool:
|
||||
"""Return whether the tensors match the tuned gfx950 fast path."""
|
||||
"""Return whether the tensors can use the gfx950 fast path."""
|
||||
try:
|
||||
_validate_inputs(hidden_states, router_weight)
|
||||
except (RuntimeError, ValueError):
|
||||
@@ -96,26 +115,30 @@ def _validate_inputs(
|
||||
raise ValueError("hidden_states and router_weight must be 2D tensors")
|
||||
if hidden_states.dtype not in (torch.bfloat16, torch.float32):
|
||||
raise ValueError("hidden_states must have dtype bfloat16 or float32")
|
||||
if router_weight.dtype != torch.float32:
|
||||
raise ValueError("router_weight must have dtype float32")
|
||||
if router_weight.dtype not in (torch.float32, torch.bfloat16):
|
||||
raise ValueError("router_weight must have dtype float32 or bfloat16")
|
||||
if hidden_states.device.type != "cuda" or router_weight.device.type != "cuda":
|
||||
raise ValueError("hidden_states and router_weight must be GPU tensors")
|
||||
if hidden_states.device != router_weight.device:
|
||||
raise ValueError("hidden_states and router_weight must be on the same device")
|
||||
if not hidden_states.is_contiguous() or not router_weight.is_contiguous():
|
||||
raise ValueError("hidden_states and router_weight must be contiguous")
|
||||
shape = (hidden_states.shape[1], router_weight.shape[0])
|
||||
if (
|
||||
shape not in ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES
|
||||
or router_weight.shape[1] != shape[0]
|
||||
):
|
||||
raise ValueError(
|
||||
"supported (hidden_size, num_experts) shape pairs are "
|
||||
"(3072, 256), (4096, 8), (4096, 192), (6144, 128), "
|
||||
"and (6144, 256)"
|
||||
)
|
||||
if router_weight.shape[1] != hidden_states.shape[1]:
|
||||
raise ValueError("router_weight and hidden_states hidden sizes differ")
|
||||
if not 0 <= hidden_states.shape[0] <= _MAX_TOKENS:
|
||||
raise ValueError(f"num_tokens must be in [0, {_MAX_TOKENS}]")
|
||||
if hidden_states.shape[0] * router_weight.numel() >= _MAX_MACS:
|
||||
raise ValueError(
|
||||
f"num_tokens * num_experts * hidden_size must be < {_MAX_MACS}"
|
||||
)
|
||||
if (
|
||||
router_weight.dtype == torch.bfloat16
|
||||
and hidden_states.shape[0] * router_weight.shape[0] > _MAX_BF16_TOKEN_EXPERTS
|
||||
):
|
||||
raise ValueError(
|
||||
f"num_tokens * num_experts must be <= {_MAX_BF16_TOKEN_EXPERTS} "
|
||||
"for bfloat16 weights"
|
||||
)
|
||||
|
||||
|
||||
def rocm_fp32_router_gemm(
|
||||
@@ -131,7 +154,10 @@ def rocm_fp32_router_gemm(
|
||||
if num_tokens == 0:
|
||||
return output
|
||||
|
||||
block_m, block_k, num_warps = _launch_config(hidden_size, num_experts, num_tokens)
|
||||
block_m, block_k, num_warps = _launch_config(
|
||||
num_experts, num_tokens, router_weight.dtype == torch.bfloat16
|
||||
)
|
||||
block_k = min(block_k, triton.next_power_of_2(hidden_size))
|
||||
grid = (triton.cdiv(num_tokens, block_m) * num_experts,)
|
||||
_rocm_fp32_router_gemm_kernel[grid](
|
||||
hidden_states,
|
||||
@@ -142,6 +168,7 @@ def rocm_fp32_router_gemm(
|
||||
N=num_experts,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_K=block_k,
|
||||
EVEN_K=hidden_size % block_k == 0,
|
||||
num_warps=num_warps,
|
||||
num_stages=1,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user