[Perf][Feat] Add generic cuteDSL LL FP32 router (GEMM) (#44343)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
6849ca696a
commit
7fad7db330
@@ -22,14 +22,16 @@ class GateLinear(ReplicatedLinear):
|
||||
"""MoE gate linear layer with multi-tier GEMM dispatch:
|
||||
|
||||
1. cuteDSL ll_bf16_gemm (SM90+, M<=16, bf16 in, fp32 out,
|
||||
K divisible by 8)
|
||||
any shape w/ K divisible by 8)
|
||||
2. fp32 specialized kernel (SM90+ or gfx950, bf16/fp32 in, fp32 out,
|
||||
M<=32, model-specific shapes)
|
||||
3. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
|
||||
4. ROCm bf16x3 router GEMM (gfx950, bf16 in, fp32 weight, fp32 out,
|
||||
3. cuteDSL ll_fp32w_gemm (SM90+, M<=32, bf16/fp32 in, fp32 out,
|
||||
any shape)
|
||||
4. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
|
||||
5. ROCm bf16x3 router GEMM (gfx950, bf16 in, fp32 weight, fp32 out,
|
||||
M>=2048)
|
||||
5. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
|
||||
6. F.linear via ReplicatedLinear (ultimate fallback)
|
||||
6. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
|
||||
7. F.linear via ReplicatedLinear (ultimate fallback)
|
||||
|
||||
The ``out_dtype`` attribute is mutable and can be set after init
|
||||
(e.g. when the required dtype depends on the expert quantization
|
||||
@@ -172,20 +174,50 @@ class GateLinear(ReplicatedLinear):
|
||||
self.quant_method = _GateLinearMethod()
|
||||
logger.info_once("Enabled ROCm BF16x3 router GEMM.")
|
||||
|
||||
# cuteDSL ll_bf16_gemm eligibility. Any dims supported, but SM90+ required bc:
|
||||
# cuteDSL ll_bf16_gemm/ll_fp32w_gemm eligibility. Any dims supported, but
|
||||
# SM90+ required because:
|
||||
# 1. PDL support. Both dot-product and split-K kernels.
|
||||
# 2. Thread Block Clusters. Split-K kernel for cross-CTA reduction.
|
||||
self.allow_ll_bf16_gemm = False
|
||||
self.allow_ll_fp32w_gemm = False
|
||||
if can_use_specialized_kernels:
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
|
||||
is_available,
|
||||
is_available as is_ll_bf16_gemm_available,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
|
||||
is_available as is_ll_fp32w_gemm_available,
|
||||
)
|
||||
|
||||
self.allow_ll_bf16_gemm = (
|
||||
self.weight.dtype == torch.bfloat16
|
||||
is_ll_bf16_gemm_available()
|
||||
and self.weight.dtype == torch.bfloat16
|
||||
and self.out_dtype == torch.float32
|
||||
and is_available()
|
||||
)
|
||||
self.allow_ll_fp32w_gemm = (
|
||||
is_ll_fp32w_gemm_available()
|
||||
and self.weight.dtype == torch.float32
|
||||
and self.out_dtype == torch.float32
|
||||
)
|
||||
|
||||
self._register_ll_fp32w_warmup()
|
||||
|
||||
def _register_ll_fp32w_warmup(self) -> None:
|
||||
# Based on multi-tier GEMM dispatch. See above.
|
||||
# This avoids compiling LL_FP32W_GEMM_KERNEL if not used.
|
||||
if not self.allow_ll_fp32w_gemm or self.allow_fp32_router_gemm:
|
||||
return
|
||||
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
|
||||
LL_FP32W_GEMM_KERNEL,
|
||||
)
|
||||
|
||||
# FP16 remains runtime-supported; JIT monitoring reports unexpected use.
|
||||
# The kernel supports it, but there is no FP16 activation use case yet.
|
||||
LL_FP32W_GEMM_KERNEL.register_warmup(
|
||||
shapes=((self.weight.shape[1], self.weight.shape[0]),),
|
||||
m_values=range(1, self.FP32_MAX_TOKENS + 1),
|
||||
a_dtypes=(torch.bfloat16, torch.float32),
|
||||
)
|
||||
|
||||
def set_out_dtype(self, out_dtype: torch.dtype) -> None:
|
||||
"""Set output dtype for the router logits after init.
|
||||
@@ -211,14 +243,23 @@ class GateLinear(ReplicatedLinear):
|
||||
|
||||
if self.allow_specialized_router_gemm:
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
|
||||
is_available,
|
||||
is_available as is_ll_bf16_gemm_available,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
|
||||
is_available as is_ll_fp32w_gemm_available,
|
||||
)
|
||||
|
||||
self.allow_ll_bf16_gemm = (
|
||||
self.weight.dtype == torch.bfloat16
|
||||
is_ll_bf16_gemm_available()
|
||||
and self.weight.dtype == torch.bfloat16
|
||||
and out_dtype == torch.float32
|
||||
and is_available()
|
||||
)
|
||||
self.allow_ll_fp32w_gemm = (
|
||||
is_ll_fp32w_gemm_available()
|
||||
and self.weight.dtype == torch.float32
|
||||
and out_dtype == torch.float32
|
||||
)
|
||||
self._register_ll_fp32w_warmup()
|
||||
|
||||
def _return(
|
||||
self, output: torch.Tensor
|
||||
@@ -228,7 +269,8 @@ class GateLinear(ReplicatedLinear):
|
||||
def forward(
|
||||
self, x: torch.Tensor
|
||||
) -> torch.Tensor | tuple[torch.Tensor, Parameter | None]:
|
||||
# Tier 1: cuteDSL ll_bf16_gemm (SM90+, any dims)
|
||||
# Tier 1: cuteDSL ll_bf16_gemm (SM90+, bf16 in, fp32 out,
|
||||
# any dims w/ K divisible by 8)
|
||||
if (
|
||||
self.allow_ll_bf16_gemm
|
||||
and x.shape[0] <= self.LL_BF16_MAX_TOKENS
|
||||
@@ -241,7 +283,7 @@ class GateLinear(ReplicatedLinear):
|
||||
output = ll_bf16_gemm(x, self.weight)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 4: ROCm bf16x3 router GEMM. Checked before tier 2 because both
|
||||
# Tier 5: ROCm bf16x3 router GEMM. Checked before tier 2 because both
|
||||
# tiers claim the same gfx950 shapes and tier 2 returns
|
||||
# unconditionally; the custom op below picks between them on the
|
||||
# runtime num_tokens.
|
||||
@@ -270,7 +312,20 @@ class GateLinear(ReplicatedLinear):
|
||||
)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 3: bf16x3 CuteDSL kernel for fp32 router weights
|
||||
# Tier 3: cuteDSL ll_fp32w_gemm (SM90+, bf16/fp32 in, fp32 out)
|
||||
if (
|
||||
self.allow_ll_fp32w_gemm
|
||||
and x.shape[0] <= 32
|
||||
and x.dtype in (torch.bfloat16, torch.float16, torch.float32)
|
||||
):
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
|
||||
LL_FP32W_GEMM_KERNEL,
|
||||
)
|
||||
|
||||
output = LL_FP32W_GEMM_KERNEL(x, self.weight)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 4: bf16x3 CuteDSL kernel for fp32 router weights
|
||||
if self.allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
|
||||
from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501
|
||||
bf16x3_router_gemm,
|
||||
@@ -279,12 +334,12 @@ class GateLinear(ReplicatedLinear):
|
||||
output = bf16x3_router_gemm(x, self.weight)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 5: cuBLAS bf16→fp32
|
||||
# Tier 6: cuBLAS bf16→fp32
|
||||
if self.allow_cublas_router_gemm and x.dtype == torch.bfloat16:
|
||||
output = torch.mm(x, self.weight.T, out_dtype=torch.float32)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 6: F.linear (ReplicatedLinear)
|
||||
# Tier 7: F.linear (ReplicatedLinear)
|
||||
if (
|
||||
self.out_dtype is not None
|
||||
and self.is_unquantized
|
||||
|
||||
Reference in New Issue
Block a user