[Perf][Feat] Add generic cuteDSL LL FP32 router (GEMM) (#44343)

Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Roberto L. Castro
2026-10-13 12:38:46 -04:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 6849ca696a
commit 7fad7db330
7 changed files with 1447 additions and 101 deletions
@@ -22,14 +22,16 @@ class GateLinear(ReplicatedLinear):
"""MoE gate linear layer with multi-tier GEMM dispatch:
1. cuteDSL ll_bf16_gemm (SM90+, M<=16, bf16 in, fp32 out,
K divisible by 8)
any shape w/ K divisible by 8)
2. fp32 specialized kernel (SM90+ or gfx950, bf16/fp32 in, fp32 out,
M<=32, model-specific shapes)
3. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
4. ROCm bf16x3 router GEMM (gfx950, bf16 in, fp32 weight, fp32 out,
3. cuteDSL ll_fp32w_gemm (SM90+, M<=32, bf16/fp32 in, fp32 out,
any shape)
4. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
5. ROCm bf16x3 router GEMM (gfx950, bf16 in, fp32 weight, fp32 out,
M>=2048)
5. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
6. F.linear via ReplicatedLinear (ultimate fallback)
6. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
7. F.linear via ReplicatedLinear (ultimate fallback)
The ``out_dtype`` attribute is mutable and can be set after init
(e.g. when the required dtype depends on the expert quantization
@@ -172,20 +174,50 @@ class GateLinear(ReplicatedLinear):
self.quant_method = _GateLinearMethod()
logger.info_once("Enabled ROCm BF16x3 router GEMM.")
# cuteDSL ll_bf16_gemm eligibility. Any dims supported, but SM90+ required bc:
# cuteDSL ll_bf16_gemm/ll_fp32w_gemm eligibility. Any dims supported, but
# SM90+ required because:
# 1. PDL support. Both dot-product and split-K kernels.
# 2. Thread Block Clusters. Split-K kernel for cross-CTA reduction.
self.allow_ll_bf16_gemm = False
self.allow_ll_fp32w_gemm = False
if can_use_specialized_kernels:
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
is_available,
is_available as is_ll_bf16_gemm_available,
)
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
is_available as is_ll_fp32w_gemm_available,
)
self.allow_ll_bf16_gemm = (
self.weight.dtype == torch.bfloat16
is_ll_bf16_gemm_available()
and self.weight.dtype == torch.bfloat16
and self.out_dtype == torch.float32
and is_available()
)
self.allow_ll_fp32w_gemm = (
is_ll_fp32w_gemm_available()
and self.weight.dtype == torch.float32
and self.out_dtype == torch.float32
)
self._register_ll_fp32w_warmup()
def _register_ll_fp32w_warmup(self) -> None:
# Based on multi-tier GEMM dispatch. See above.
# This avoids compiling LL_FP32W_GEMM_KERNEL if not used.
if not self.allow_ll_fp32w_gemm or self.allow_fp32_router_gemm:
return
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
LL_FP32W_GEMM_KERNEL,
)
# FP16 remains runtime-supported; JIT monitoring reports unexpected use.
# The kernel supports it, but there is no FP16 activation use case yet.
LL_FP32W_GEMM_KERNEL.register_warmup(
shapes=((self.weight.shape[1], self.weight.shape[0]),),
m_values=range(1, self.FP32_MAX_TOKENS + 1),
a_dtypes=(torch.bfloat16, torch.float32),
)
def set_out_dtype(self, out_dtype: torch.dtype) -> None:
"""Set output dtype for the router logits after init.
@@ -211,14 +243,23 @@ class GateLinear(ReplicatedLinear):
if self.allow_specialized_router_gemm:
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
is_available,
is_available as is_ll_bf16_gemm_available,
)
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
is_available as is_ll_fp32w_gemm_available,
)
self.allow_ll_bf16_gemm = (
self.weight.dtype == torch.bfloat16
is_ll_bf16_gemm_available()
and self.weight.dtype == torch.bfloat16
and out_dtype == torch.float32
and is_available()
)
self.allow_ll_fp32w_gemm = (
is_ll_fp32w_gemm_available()
and self.weight.dtype == torch.float32
and out_dtype == torch.float32
)
self._register_ll_fp32w_warmup()
def _return(
self, output: torch.Tensor
@@ -228,7 +269,8 @@ class GateLinear(ReplicatedLinear):
def forward(
self, x: torch.Tensor
) -> torch.Tensor | tuple[torch.Tensor, Parameter | None]:
# Tier 1: cuteDSL ll_bf16_gemm (SM90+, any dims)
# Tier 1: cuteDSL ll_bf16_gemm (SM90+, bf16 in, fp32 out,
# any dims w/ K divisible by 8)
if (
self.allow_ll_bf16_gemm
and x.shape[0] <= self.LL_BF16_MAX_TOKENS
@@ -241,7 +283,7 @@ class GateLinear(ReplicatedLinear):
output = ll_bf16_gemm(x, self.weight)
return self._return(output)
# Tier 4: ROCm bf16x3 router GEMM. Checked before tier 2 because both
# Tier 5: ROCm bf16x3 router GEMM. Checked before tier 2 because both
# tiers claim the same gfx950 shapes and tier 2 returns
# unconditionally; the custom op below picks between them on the
# runtime num_tokens.
@@ -270,7 +312,20 @@ class GateLinear(ReplicatedLinear):
)
return self._return(output)
# Tier 3: bf16x3 CuteDSL kernel for fp32 router weights
# Tier 3: cuteDSL ll_fp32w_gemm (SM90+, bf16/fp32 in, fp32 out)
if (
self.allow_ll_fp32w_gemm
and x.shape[0] <= 32
and x.dtype in (torch.bfloat16, torch.float16, torch.float32)
):
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
LL_FP32W_GEMM_KERNEL,
)
output = LL_FP32W_GEMM_KERNEL(x, self.weight)
return self._return(output)
# Tier 4: bf16x3 CuteDSL kernel for fp32 router weights
if self.allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501
bf16x3_router_gemm,
@@ -279,12 +334,12 @@ class GateLinear(ReplicatedLinear):
output = bf16x3_router_gemm(x, self.weight)
return self._return(output)
# Tier 5: cuBLAS bf16→fp32
# Tier 6: cuBLAS bf16→fp32
if self.allow_cublas_router_gemm and x.dtype == torch.bfloat16:
output = torch.mm(x, self.weight.T, out_dtype=torch.float32)
return self._return(output)
# Tier 6: F.linear (ReplicatedLinear)
# Tier 7: F.linear (ReplicatedLinear)
if (
self.out_dtype is not None
and self.is_unquantized