[Perf][Feat] Add generic cuteDSL LL FP32 router (GEMM) (#44343)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
6849ca696a
commit
7fad7db330
@@ -0,0 +1,457 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import const_expr
|
||||
|
||||
|
||||
class LLFp32WDotprod:
|
||||
"""FP32-weight router GEMM kernel based on CTA-local dot products.
|
||||
|
||||
This kernel computes C[M, N] = A[M, K] @ B[N, K]^T for bf16/fp16/fp32
|
||||
activations, fp32 weights, and fp32 output. It launches one CTA per
|
||||
small output-column tile, distributes K across CTA threads with vectorized
|
||||
loads, accumulates one fp32 dot product per token/expert, and reduces
|
||||
through warp shuffles plus shared memory.
|
||||
|
||||
:param k: Compile-time K dimension specialized into the generated kernel.
|
||||
:type k: int
|
||||
:param bs: Threads per CTA and K-stripe width used by the reduction.
|
||||
:type bs: int
|
||||
:param main_vec_width: Elements loaded per thread in the main loop.
|
||||
:type main_vec_width: int
|
||||
:param tail_vec_width: Elements loaded per thread in the vector tail.
|
||||
:type tail_vec_width: int
|
||||
:param use_pdl: Whether to launch with Programmatic Dependent Launch.
|
||||
:type use_pdl: bool
|
||||
|
||||
:note: Supported A/B data types:
|
||||
- BFloat16/Float32
|
||||
- Float16/Float32
|
||||
- Float32/Float32
|
||||
:note: Supported accumulator data types:
|
||||
- Float32
|
||||
:note: Supported C data types:
|
||||
- Float32
|
||||
:note: Constraints:
|
||||
- Inputs must be contiguous row-major 2D tensors.
|
||||
|
||||
:note: K is handled as vectorized main/tail loops plus scalar remainder.
|
||||
|
||||
:compile-key: ``(M, K, bs, epb, act_dtype)`` selects the token count,
|
||||
hidden size, CTA thread/K-stripe width, experts per CTA, and activation
|
||||
input type.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
m: int,
|
||||
k: int,
|
||||
bs: int = 128,
|
||||
main_vec_width: int = 8,
|
||||
tail_vec_width: int = 4,
|
||||
token_groups: int = 1,
|
||||
epb: int = 1,
|
||||
use_pdl: bool = False,
|
||||
):
|
||||
"""Initialize the dot-product kernel configuration.
|
||||
|
||||
This configuration fixes the CTA thread count, reduction warp count,
|
||||
vector widths, and K-loop decomposition used by the generated kernel.
|
||||
|
||||
:param m: Token count M selected by the compile key.
|
||||
:type m: int
|
||||
:param k: Hidden size K used to specialize the K-loop decomposition.
|
||||
:type k: int
|
||||
:param bs: Threads per CTA and K-stripe width for the reduction.
|
||||
:type bs: int
|
||||
:param main_vec_width: Elements loaded per thread in the main loop.
|
||||
:type main_vec_width: int
|
||||
:param tail_vec_width: Elements loaded per thread in the vector tail.
|
||||
:type tail_vec_width: int
|
||||
:param use_pdl: Whether to launch with Programmatic Dependent Launch.
|
||||
:type use_pdl: bool
|
||||
"""
|
||||
self.m = m
|
||||
self.m_per_group = m // token_groups
|
||||
self.bs = bs
|
||||
self.main_vec_width = main_vec_width
|
||||
self.tail_vec_width = tail_vec_width
|
||||
self.token_groups = token_groups
|
||||
self.epb = epb
|
||||
self.use_pdl = use_pdl
|
||||
self.num_warps = bs // cute.arch.WARP_SIZE
|
||||
self._init_k_tiles(k)
|
||||
|
||||
def _vectorized_elems(self, k_extent: int, vec_width: int) -> int:
|
||||
vector_tile = vec_width * self.bs
|
||||
return (k_extent // vector_tile) * vector_tile
|
||||
|
||||
def _init_k_tiles(self, k: int) -> None:
|
||||
"""Split K into vector loops, scalar rounds, and ragged tail."""
|
||||
self.k_main_elems = self._vectorized_elems(k, self.main_vec_width)
|
||||
self.k_after_main = k - self.k_main_elems
|
||||
self.k_tail_elems = self._vectorized_elems(
|
||||
self.k_after_main, self.tail_vec_width
|
||||
)
|
||||
self.k_done_all = self.k_main_elems + self.k_tail_elems
|
||||
self.scalar_rem = k - self.k_done_all
|
||||
self.ks_full = self.scalar_rem // self.bs
|
||||
self.ks_part = self.scalar_rem % self.bs
|
||||
self.k_scalar_full = self.ks_full * self.bs
|
||||
self.k_part_offset = self.k_done_all + self.k_scalar_full
|
||||
self.main_tiles = self.k_main_elems // (self.main_vec_width * self.bs)
|
||||
self.tail_tiles = self.k_tail_elems // (self.tail_vec_width * self.bs)
|
||||
|
||||
@cute.jit
|
||||
def _vector_dotprod(
|
||||
self,
|
||||
acc: cute.Tensor,
|
||||
tA: cute.Tensor,
|
||||
tB: cute.Tensor,
|
||||
M: cutlass.Constexpr,
|
||||
epb: cutlass.Constexpr,
|
||||
num_tiles: cutlass.Constexpr,
|
||||
):
|
||||
for tile in cutlass.range_constexpr(num_tiles):
|
||||
bt0 = tB[0, None, tile]
|
||||
br0 = cute.make_rmem_tensor_like(bt0)
|
||||
cute.autovec_copy(bt0, br0)
|
||||
br0_f32 = br0.load().to(cutlass.Float32)
|
||||
|
||||
if const_expr(epb == 2):
|
||||
bt1 = tB[1, None, tile]
|
||||
br1 = cute.make_rmem_tensor_like(bt1)
|
||||
cute.autovec_copy(bt1, br1)
|
||||
br1_f32 = br1.load().to(cutlass.Float32)
|
||||
|
||||
for m in cutlass.range_constexpr(M):
|
||||
at = tA[m, None, tile]
|
||||
ar = cute.make_rmem_tensor_like(at)
|
||||
cute.autovec_copy(at, ar)
|
||||
vec_width: cutlass.Constexpr = cute.size(ar)
|
||||
for v in cutlass.range_constexpr(vec_width):
|
||||
av = ar[v].to(cutlass.Float32)
|
||||
acc[m, 0] = acc[m, 0] + av * br0_f32[v]
|
||||
if const_expr(epb == 2):
|
||||
acc[m, 1] = acc[m, 1] + av * br1_f32[v]
|
||||
|
||||
def _make_thread_vector_slice(
|
||||
self,
|
||||
gA_vec: cute.Tensor,
|
||||
gB_vec: cute.Tensor,
|
||||
tidx: cutlass.Int32,
|
||||
bs: cutlass.Constexpr,
|
||||
):
|
||||
# A: (M, K_TILE, K_LANE, K_VEC); B: (epb, K_TILE, K_LANE, K_VEC).
|
||||
tA = cute.logical_divide(gA_vec, (None, (None, bs)))
|
||||
tB = cute.logical_divide(gB_vec, (None, (None, bs)))
|
||||
return tA[None, (None, (tidx, None))], tB[None, (None, (tidx, None))]
|
||||
|
||||
def _make_m_slice(
|
||||
self,
|
||||
gX: cute.Tensor,
|
||||
m_offset: cutlass.Int32,
|
||||
m_extent: cutlass.Constexpr,
|
||||
):
|
||||
return cute.local_tile(
|
||||
cute.domain_offset((m_offset, 0), gX),
|
||||
(m_extent, cute.size(gX, mode=[1])),
|
||||
(0, 0),
|
||||
)
|
||||
|
||||
def _make_n_tile(
|
||||
self,
|
||||
gX: cute.Tensor,
|
||||
n_tile_idx: cutlass.Int32,
|
||||
n_extent: cutlass.Constexpr,
|
||||
):
|
||||
return cute.local_tile(
|
||||
gX,
|
||||
(n_extent, cute.size(gX, mode=[1])),
|
||||
(n_tile_idx, 0),
|
||||
)
|
||||
|
||||
def _make_sm_m_slice(
|
||||
self,
|
||||
sm: cute.Tensor,
|
||||
m_offset: cutlass.Int32,
|
||||
m_extent: cutlass.Constexpr,
|
||||
epb: cutlass.Constexpr,
|
||||
num_warps: cutlass.Constexpr,
|
||||
):
|
||||
return cute.local_tile(
|
||||
cute.domain_offset((m_offset, 0, 0), sm),
|
||||
(m_extent, epb, num_warps),
|
||||
(0, 0, 0),
|
||||
)
|
||||
|
||||
def _make_k_slice(
|
||||
self,
|
||||
gX: cute.Tensor,
|
||||
k_offset: cutlass.Constexpr,
|
||||
k_extent: cutlass.Constexpr,
|
||||
):
|
||||
k_layout_extent: cutlass.Constexpr = (
|
||||
1 if const_expr(k_extent == 0) else k_extent
|
||||
)
|
||||
|
||||
if const_expr(k_offset == 0):
|
||||
return cute.local_tile(
|
||||
gX, (cute.size(gX, mode=[0]), k_layout_extent), (0, 0)
|
||||
)
|
||||
return cute.local_tile(
|
||||
cute.domain_offset((0, k_offset), gX),
|
||||
(cute.size(gX, mode=[0]), k_layout_extent),
|
||||
(0, 0),
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def _compute_token_group(
|
||||
self,
|
||||
gA: cute.Tensor,
|
||||
gB: cute.Tensor,
|
||||
sm: cute.Tensor,
|
||||
n_tile_idx: cutlass.Int32,
|
||||
tidx: cutlass.Int32,
|
||||
local_wid: cutlass.Int32,
|
||||
m_per_group: cutlass.Constexpr,
|
||||
m_offset: cutlass.Int32,
|
||||
main_vec_width: cutlass.Constexpr,
|
||||
tail_vec_width: cutlass.Constexpr,
|
||||
bs: cutlass.Constexpr,
|
||||
epb: cutlass.Constexpr,
|
||||
num_warps: cutlass.Constexpr,
|
||||
k_main_elems: cutlass.Constexpr,
|
||||
k_tail_elems: cutlass.Constexpr,
|
||||
k_done_all: cutlass.Constexpr,
|
||||
ks_full: cutlass.Constexpr,
|
||||
ks_part: cutlass.Constexpr,
|
||||
k_scalar_full: cutlass.Constexpr,
|
||||
k_part_offset: cutlass.Constexpr,
|
||||
main_tiles: cutlass.Constexpr,
|
||||
tail_tiles: cutlass.Constexpr,
|
||||
):
|
||||
# Each CTA reuses A for epb adjacent experts.
|
||||
acc = cute.make_rmem_tensor((m_per_group, epb), cutlass.Float32)
|
||||
acc.fill(0.0)
|
||||
|
||||
gA_group = self._make_m_slice(gA, m_offset, m_per_group)
|
||||
gB_epb = self._make_n_tile(gB, n_tile_idx, epb)
|
||||
sm_group = self._make_sm_m_slice(sm, m_offset, m_per_group, epb, num_warps)
|
||||
|
||||
if const_expr(k_main_elems > 0):
|
||||
gA_main = self._make_k_slice(gA_group, 0, k_main_elems)
|
||||
gB_main = self._make_k_slice(gB_epb, 0, k_main_elems)
|
||||
gA_vec = cute.logical_divide(gA_main, (None, main_vec_width))
|
||||
gB_vec = cute.logical_divide(gB_main, (None, main_vec_width))
|
||||
tA, tB = self._make_thread_vector_slice(gA_vec, gB_vec, tidx, bs)
|
||||
self._vector_dotprod(acc, tA, tB, m_per_group, epb, main_tiles)
|
||||
|
||||
if const_expr(k_tail_elems > 0):
|
||||
gA_tail = self._make_k_slice(gA_group, k_main_elems, k_tail_elems)
|
||||
gB_tail = self._make_k_slice(gB_epb, k_main_elems, k_tail_elems)
|
||||
gA_tail_vec = cute.logical_divide(gA_tail, (None, tail_vec_width))
|
||||
gB_tail_vec = cute.logical_divide(gB_tail, (None, tail_vec_width))
|
||||
tA_t, tB_t = self._make_thread_vector_slice(
|
||||
gA_tail_vec, gB_tail_vec, tidx, bs
|
||||
)
|
||||
self._vector_dotprod(acc, tA_t, tB_t, m_per_group, epb, tail_tiles)
|
||||
|
||||
if const_expr(ks_full > 0):
|
||||
gA_scalar = self._make_k_slice(gA_group, k_done_all, k_scalar_full)
|
||||
gB_scalar = self._make_k_slice(gB_epb, k_done_all, k_scalar_full)
|
||||
gA_scalar_vec = cute.logical_divide(gA_scalar, (None, 1))
|
||||
gB_scalar_vec = cute.logical_divide(gB_scalar, (None, 1))
|
||||
tA_s, tB_s = self._make_thread_vector_slice(
|
||||
gA_scalar_vec, gB_scalar_vec, tidx, bs
|
||||
)
|
||||
self._vector_dotprod(acc, tA_s, tB_s, m_per_group, epb, ks_full)
|
||||
|
||||
if const_expr(ks_part > 0):
|
||||
gA_part = self._make_k_slice(gA_group, k_part_offset, ks_part)
|
||||
gB_part = self._make_k_slice(gB_epb, k_part_offset, ks_part)
|
||||
if tidx < ks_part:
|
||||
for local_m in cutlass.range_constexpr(m_per_group):
|
||||
av = gA_part[local_m, tidx].to(cutlass.Float32)
|
||||
for e in cutlass.range_constexpr(epb):
|
||||
bv = gB_part[e, tidx].to(cutlass.Float32)
|
||||
acc[local_m, e] = acc[local_m, e] + av * bv
|
||||
|
||||
for local_m in cutlass.range_constexpr(m_per_group):
|
||||
for e in cutlass.range_constexpr(epb):
|
||||
acc[local_m, e] = cute.arch.warp_reduction_sum(acc[local_m, e])
|
||||
|
||||
with cute.arch.elect_one():
|
||||
for local_m in cutlass.range_constexpr(m_per_group):
|
||||
for e in cutlass.range_constexpr(epb):
|
||||
sm_group[local_m, e, local_wid] = acc[local_m, e]
|
||||
|
||||
@cute.jit
|
||||
def _store_outputs(
|
||||
self,
|
||||
gC: cute.Tensor,
|
||||
sm: cute.Tensor,
|
||||
n_tile_idx: cutlass.Int32,
|
||||
N_dim: cutlass.Int32,
|
||||
tidx: cutlass.Int32,
|
||||
group_idx: cutlass.Int32,
|
||||
M: cutlass.Constexpr,
|
||||
epb: cutlass.Constexpr,
|
||||
):
|
||||
n_tile_layout = cute.make_layout(
|
||||
(cute.ceil_div(N_dim, epb), epb), stride=(epb, 1)
|
||||
)
|
||||
n_coords = cute.make_tensor(0, n_tile_layout)
|
||||
|
||||
if group_idx == 0 and tidx < M:
|
||||
for e in cutlass.range_constexpr(epb):
|
||||
n = n_coords[n_tile_idx, e]
|
||||
if n < N_dim:
|
||||
partials = sm[tidx, e, None].load()
|
||||
gC[tidx, n] = partials.reduce(
|
||||
cute.ReductionOp.ADD,
|
||||
init_val=cutlass.Float32(0.0),
|
||||
reduction_profile=0,
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
gA: cute.Tensor,
|
||||
gB: cute.Tensor,
|
||||
gC: cute.Tensor,
|
||||
N_dim: cutlass.Int32,
|
||||
stream: CUstream,
|
||||
):
|
||||
"""Execute the dot-product GEMM operation in steps:
|
||||
- Launch one CTA per small output-column tile with ``bs`` threads.
|
||||
- Keep one FP32 accumulator per token/expert in each thread.
|
||||
- Traverse K with vectorized main/tail, scalar, and ragged-tail loops
|
||||
from the precomputed K decomposition.
|
||||
- Reduce each token accumulator first within the warp, then across
|
||||
warps through shared memory, and store ``C[:, n]``.
|
||||
|
||||
:param gA: Input tensor A with shape ``[M, K]``.
|
||||
:type gA: cute.Tensor
|
||||
:param gB: Input tensor B with shape ``[N, K]``.
|
||||
:type gB: cute.Tensor
|
||||
:param gC: Output tensor C with shape ``[M, N]``.
|
||||
:type gC: cute.Tensor
|
||||
:param N_dim: Output column count used for the launch grid.
|
||||
:type N_dim: cutlass.Int32
|
||||
:param stream: CUDA stream for asynchronous execution.
|
||||
:type stream: CUstream
|
||||
"""
|
||||
self.kernel(
|
||||
gA,
|
||||
gB,
|
||||
gC,
|
||||
N_dim,
|
||||
self.m,
|
||||
self.m_per_group,
|
||||
self.main_vec_width,
|
||||
self.tail_vec_width,
|
||||
self.bs,
|
||||
self.num_warps,
|
||||
self.token_groups,
|
||||
self.epb,
|
||||
self.k_main_elems,
|
||||
self.k_tail_elems,
|
||||
self.k_done_all,
|
||||
self.ks_full,
|
||||
self.ks_part,
|
||||
self.k_scalar_full,
|
||||
self.k_part_offset,
|
||||
self.main_tiles,
|
||||
self.tail_tiles,
|
||||
).launch(
|
||||
grid=[cute.ceil_div(N_dim, self.epb), 1, 1],
|
||||
block=[self.bs, self.token_groups, 1],
|
||||
smem=self.m * self.epb * 4 * self.num_warps,
|
||||
stream=stream,
|
||||
use_pdl=self.use_pdl,
|
||||
min_blocks_per_mp=1,
|
||||
)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
gA: cute.Tensor,
|
||||
gB: cute.Tensor,
|
||||
gC: cute.Tensor,
|
||||
N_dim: cutlass.Int32,
|
||||
M: cutlass.Constexpr,
|
||||
m_per_group: cutlass.Constexpr,
|
||||
main_vec_width: cutlass.Constexpr,
|
||||
tail_vec_width: cutlass.Constexpr,
|
||||
bs: cutlass.Constexpr,
|
||||
num_warps: cutlass.Constexpr,
|
||||
token_groups: cutlass.Constexpr,
|
||||
epb: cutlass.Constexpr,
|
||||
k_main_elems: cutlass.Constexpr,
|
||||
k_tail_elems: cutlass.Constexpr,
|
||||
k_done_all: cutlass.Constexpr,
|
||||
ks_full: cutlass.Constexpr,
|
||||
ks_part: cutlass.Constexpr,
|
||||
k_scalar_full: cutlass.Constexpr,
|
||||
k_part_offset: cutlass.Constexpr,
|
||||
main_tiles: cutlass.Constexpr,
|
||||
tail_tiles: cutlass.Constexpr,
|
||||
):
|
||||
tidx, group_idx, _ = cute.arch.thread_idx()
|
||||
n_tile_idx, _, _ = cute.arch.block_idx()
|
||||
|
||||
local_wid = tidx // cute.arch.WARP_SIZE
|
||||
|
||||
if const_expr(self.use_pdl):
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
smem_red_layout = cute.make_layout(
|
||||
(M, epb, num_warps), stride=(epb * num_warps, num_warps, 1)
|
||||
)
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
sm = smem.allocate_tensor(cutlass.Float32, smem_red_layout, byte_alignment=16)
|
||||
|
||||
if const_expr(token_groups == 1):
|
||||
m_offset: cutlass.Constexpr = 0
|
||||
else:
|
||||
m_offset = group_idx * m_per_group
|
||||
self._compute_token_group(
|
||||
gA,
|
||||
gB,
|
||||
sm,
|
||||
n_tile_idx,
|
||||
tidx,
|
||||
local_wid,
|
||||
m_per_group,
|
||||
m_offset,
|
||||
main_vec_width,
|
||||
tail_vec_width,
|
||||
bs,
|
||||
epb,
|
||||
num_warps,
|
||||
k_main_elems,
|
||||
k_tail_elems,
|
||||
k_done_all,
|
||||
ks_full,
|
||||
ks_part,
|
||||
k_scalar_full,
|
||||
k_part_offset,
|
||||
main_tiles,
|
||||
tail_tiles,
|
||||
)
|
||||
|
||||
cute.arch.sync_threads()
|
||||
self._store_outputs(gC, sm, n_tile_idx, N_dim, tidx, group_idx, M, epb)
|
||||
|
||||
if const_expr(self.use_pdl):
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
|
||||
|
||||
def make_host_fp32w(
|
||||
k_val: int, bs: int = 128, token_groups: int = 1, epb: int = 1, m: int = 1
|
||||
):
|
||||
return LLFp32WDotprod(m=m, k=k_val, bs=bs, token_groups=token_groups, epb=epb)
|
||||
@@ -0,0 +1,120 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_cutedsl_available: bool | None = None
|
||||
_cute_ctx: tuple[Any, Any] | None = None
|
||||
|
||||
|
||||
def is_cutedsl_available(kernel_name: str) -> bool:
|
||||
global _cutedsl_available
|
||||
if _cutedsl_available is not None:
|
||||
return _cutedsl_available
|
||||
try:
|
||||
import cutlass # noqa: F401
|
||||
import cutlass.cute # noqa: F401
|
||||
|
||||
_cutedsl_available = True
|
||||
except ImportError:
|
||||
_cutedsl_available = False
|
||||
logger.info(
|
||||
"cuteDSL (CUTLASS Python) not available, %s disabled",
|
||||
kernel_name,
|
||||
)
|
||||
return _cutedsl_available
|
||||
|
||||
|
||||
def cute_context():
|
||||
global _cute_ctx
|
||||
if _cute_ctx is not None:
|
||||
return _cute_ctx
|
||||
import cutlass.cute as cute
|
||||
from cuda.bindings.driver import CUstream
|
||||
|
||||
_cute_ctx = (cute, CUstream)
|
||||
return _cute_ctx
|
||||
|
||||
|
||||
def current_cuda_stream():
|
||||
_, CUstream = cute_context()
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
|
||||
return CUstream(current_stream().cuda_stream)
|
||||
|
||||
|
||||
def use_pdl() -> bool:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
return current_platform.is_arch_support_pdl()
|
||||
|
||||
|
||||
def cutlass_dtype(dtype: torch.dtype):
|
||||
from cutlass import BFloat16, Float16, Float32
|
||||
|
||||
if dtype == torch.bfloat16:
|
||||
return BFloat16
|
||||
if dtype == torch.float16:
|
||||
return Float16
|
||||
if dtype == torch.float32:
|
||||
return Float32
|
||||
raise ValueError(f"unsupported router GEMM dtype: {dtype}")
|
||||
|
||||
|
||||
def make_fake_gemm_tensors(
|
||||
*,
|
||||
M,
|
||||
K,
|
||||
N,
|
||||
a_dtype: torch.dtype,
|
||||
b_dtype: torch.dtype,
|
||||
divisibility: int,
|
||||
):
|
||||
from cutlass import Float32
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
hidden_states = make_fake_tensor(
|
||||
cutlass_dtype(a_dtype), (M, K), divisibility=divisibility
|
||||
)
|
||||
router_weight = make_fake_tensor(
|
||||
cutlass_dtype(b_dtype), (N, K), divisibility=divisibility
|
||||
)
|
||||
output = make_fake_tensor(Float32, (M, N), divisibility=1)
|
||||
return hidden_states, router_weight, output
|
||||
|
||||
|
||||
def validate_common_gemm_inputs(
|
||||
hidden_states: torch.Tensor,
|
||||
router_weight: torch.Tensor,
|
||||
output_dtype: torch.dtype,
|
||||
*,
|
||||
op_name: str,
|
||||
k_multiple: int | None = None,
|
||||
) -> None:
|
||||
if hidden_states.dim() != 2 or router_weight.dim() != 2:
|
||||
raise ValueError("hidden_states and router_weight must be 2D tensors")
|
||||
if hidden_states.device.type != "cuda" or router_weight.device.type != "cuda":
|
||||
raise ValueError("hidden_states and router_weight must have device_type=cuda")
|
||||
if hidden_states.device != router_weight.device:
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must be on the same CUDA device"
|
||||
)
|
||||
if output_dtype != torch.float32:
|
||||
raise ValueError(f"{op_name} only supports output_dtype=torch.float32")
|
||||
if hidden_states.shape[1] != router_weight.shape[1]:
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must have matching K dimensions"
|
||||
)
|
||||
if k_multiple is not None and hidden_states.shape[1] % k_multiple != 0:
|
||||
raise ValueError(f"{op_name} requires K to be divisible by {k_multiple}")
|
||||
if not hidden_states.is_contiguous() or not router_weight.is_contiguous():
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must be contiguous row-major inputs"
|
||||
)
|
||||
@@ -11,24 +11,20 @@ from typing import Any, Literal
|
||||
|
||||
import torch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from ._ll_router_common import (
|
||||
current_cuda_stream,
|
||||
cute_context,
|
||||
is_cutedsl_available,
|
||||
make_fake_gemm_tensors,
|
||||
use_pdl,
|
||||
validate_common_gemm_inputs,
|
||||
)
|
||||
|
||||
_cutedsl_available: bool | None = None
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
global _cutedsl_available
|
||||
if _cutedsl_available is not None:
|
||||
return _cutedsl_available
|
||||
try:
|
||||
import cutlass # noqa: F401
|
||||
import cutlass.cute # noqa: F401
|
||||
|
||||
_cutedsl_available = True
|
||||
except ImportError:
|
||||
_cutedsl_available = False
|
||||
logger.info("cuteDSL (CUTLASS Python) not available, ll_bf16_gemm disabled")
|
||||
return _cutedsl_available
|
||||
return is_cutedsl_available("ll_bf16_gemm")
|
||||
|
||||
|
||||
# Default configs
|
||||
@@ -96,33 +92,6 @@ def _arch_tuned_configs() -> tuple[
|
||||
return {}, {}
|
||||
|
||||
|
||||
_cute_ctx = None
|
||||
|
||||
|
||||
def _cute():
|
||||
global _cute_ctx
|
||||
if _cute_ctx is not None:
|
||||
return _cute_ctx
|
||||
import cutlass.cute as cute
|
||||
from cuda.bindings.driver import CUstream
|
||||
|
||||
_cute_ctx = (cute, CUstream)
|
||||
return _cute_ctx
|
||||
|
||||
|
||||
def _stream():
|
||||
_, CUstream = _cute()
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
|
||||
return CUstream(current_stream().cuda_stream)
|
||||
|
||||
|
||||
def _use_pdl() -> bool:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
return current_platform.is_arch_support_pdl()
|
||||
|
||||
|
||||
class LLBf16Gemm:
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CompileKey:
|
||||
@@ -165,16 +134,17 @@ class LLBf16Gemm:
|
||||
|
||||
@staticmethod
|
||||
def _fake_gemm_tensors(*, M, K, N, divisibility: int):
|
||||
from cutlass import BFloat16, Float32
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
hidden_states = make_fake_tensor(BFloat16, (M, K), divisibility=divisibility)
|
||||
router_weight = make_fake_tensor(BFloat16, (N, K), divisibility=divisibility)
|
||||
output = make_fake_tensor(Float32, (M, N), divisibility=1)
|
||||
return hidden_states, router_weight, output
|
||||
return make_fake_gemm_tensors(
|
||||
M=M,
|
||||
K=K,
|
||||
N=N,
|
||||
a_dtype=torch.bfloat16,
|
||||
b_dtype=torch.bfloat16,
|
||||
divisibility=divisibility,
|
||||
)
|
||||
|
||||
def _compile_splitk(self, compile_key: CompileKey) -> None:
|
||||
cute, _ = _cute()
|
||||
cute, _ = cute_context()
|
||||
from ._ll_bf16_splitk import LLBf16SplitK
|
||||
|
||||
hidden_states, router_weight, output = self._fake_gemm_tensors(
|
||||
@@ -189,14 +159,14 @@ class LLBf16Gemm:
|
||||
num_stages=compile_key.num_stages,
|
||||
num_dma_warps=4,
|
||||
split_k=compile_key.split_k,
|
||||
use_pdl=_use_pdl(),
|
||||
use_pdl=use_pdl(),
|
||||
)
|
||||
compiled = cute.compile(
|
||||
gemm,
|
||||
hidden_states,
|
||||
router_weight,
|
||||
output,
|
||||
_stream(),
|
||||
current_cuda_stream(),
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
self._splitk_cache[(compile_key.split_k, compile_key.num_stages)] = compiled
|
||||
@@ -207,7 +177,7 @@ class LLBf16Gemm:
|
||||
)
|
||||
|
||||
def _compile_dotprod(self, compile_key: CompileKey) -> None:
|
||||
cute, _ = _cute()
|
||||
cute, _ = cute_context()
|
||||
from ._ll_bf16_dotprod import LLBf16Dotprod
|
||||
|
||||
N = cute.sym_int()
|
||||
@@ -221,7 +191,7 @@ class LLBf16Gemm:
|
||||
gemm = LLBf16Dotprod(
|
||||
k=compile_key.K,
|
||||
bs=compile_key.bs,
|
||||
use_pdl=_use_pdl(),
|
||||
use_pdl=use_pdl(),
|
||||
prefetch_pdl_weights=self._prefetch_pdl_weights,
|
||||
)
|
||||
compiled = cute.compile(
|
||||
@@ -232,7 +202,7 @@ class LLBf16Gemm:
|
||||
compile_key.M,
|
||||
compile_key.K,
|
||||
1, # runtime N placeholder for fake-tensor compile
|
||||
_stream(),
|
||||
current_cuda_stream(),
|
||||
options="--enable-tvm-ffi --ptxas-options -maxrregcount=64",
|
||||
)
|
||||
self._compiled_cache[(compile_key.M, compile_key.K, compile_key.bs)] = compiled
|
||||
@@ -269,34 +239,19 @@ class LLBf16Gemm:
|
||||
router_weight: torch.Tensor,
|
||||
output_dtype: torch.dtype,
|
||||
) -> None:
|
||||
if hidden_states.dim() != 2 or router_weight.dim() != 2:
|
||||
raise ValueError("hidden_states and router_weight must be 2D tensors")
|
||||
if (
|
||||
hidden_states.dtype != torch.bfloat16
|
||||
or router_weight.dtype != torch.bfloat16
|
||||
):
|
||||
raise ValueError("hidden_states and router_weight must have dtype=bfloat16")
|
||||
if hidden_states.device.type != "cuda" or router_weight.device.type != "cuda":
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must have device_type=cuda"
|
||||
)
|
||||
if hidden_states.device != router_weight.device:
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must be on the same CUDA device"
|
||||
)
|
||||
if output_dtype != torch.float32:
|
||||
raise ValueError("ll_bf16_gemm only supports output_dtype=torch.float32")
|
||||
if hidden_states.shape[1] != router_weight.shape[1]:
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must have matching K dimensions"
|
||||
)
|
||||
# Kernels use vectorized bf16 loads and require 16-byte row alignment.
|
||||
if hidden_states.shape[1] % 8 != 0:
|
||||
raise ValueError("ll_bf16_gemm requires K to be divisible by 8")
|
||||
if not hidden_states.is_contiguous() or not router_weight.is_contiguous():
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must be contiguous row-major inputs"
|
||||
)
|
||||
validate_common_gemm_inputs(
|
||||
hidden_states,
|
||||
router_weight,
|
||||
output_dtype,
|
||||
op_name="ll_bf16_gemm",
|
||||
k_multiple=8,
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
@@ -320,7 +275,7 @@ class LLBf16Gemm:
|
||||
self.compile(compile_key)
|
||||
kernel = self._compiled_cache[dotprod_cache_key]
|
||||
|
||||
stream = _stream()
|
||||
stream = current_cuda_stream()
|
||||
output = torch.empty(M, N, dtype=output_dtype, device=hidden_states.device)
|
||||
if compile_key.backend == "splitk":
|
||||
kernel(hidden_states, router_weight, output, stream, 1.0)
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel, zip_inputs
|
||||
|
||||
from ._ll_router_common import (
|
||||
current_cuda_stream,
|
||||
cute_context,
|
||||
is_cutedsl_available,
|
||||
make_fake_gemm_tensors,
|
||||
use_pdl,
|
||||
validate_common_gemm_inputs,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
return is_cutedsl_available("ll_fp32w_gemm")
|
||||
|
||||
|
||||
_DEFAULT_DOTPROD_CONFIG = (192, 1, 1)
|
||||
_DEFAULT_DOTPROD_GROUPED_CONFIG = (192, 2, 1)
|
||||
|
||||
|
||||
_TUNED_DOTPROD_CONFIGS: dict[tuple[int, int], dict[int, tuple[int, int, int]]] = {
|
||||
(6144, 128): {
|
||||
**{
|
||||
M: (384, 1, 1)
|
||||
for M in (
|
||||
*range(1, 6),
|
||||
*range(9, 14, 2),
|
||||
17,
|
||||
*range(23, 32, 2),
|
||||
)
|
||||
},
|
||||
**{M: (256, 1, 1) for M in (7, 15, *range(19, 22, 2))},
|
||||
**{M: (384, 2, 1) for M in range(6, 12, 2)},
|
||||
**{M: (192, 2, 1) for M in (*range(12, 26, 2), 32)},
|
||||
**{M: (128, 2, 1) for M in range(26, 32, 2)},
|
||||
},
|
||||
(6144, 256): {
|
||||
**{M: (384, 1, 1) for M in range(1, 4)},
|
||||
**{M: (128, 1, 1) for M in (*range(4, 7, 2), 11)},
|
||||
**{M: (128, 1, 2) for M in range(5, 11, 2)},
|
||||
**{M: (128, 2, 2) for M in range(8, 33, 2)},
|
||||
**{M: (256, 1, 2) for M in range(13, 33, 2)},
|
||||
},
|
||||
}
|
||||
_SUPPORTED_ACTIVATION_DTYPES = (torch.bfloat16, torch.float16, torch.float32)
|
||||
|
||||
|
||||
class LLFp32WGemm(VllmJitKernel["LLFp32WGemm.CompileKey"]):
|
||||
# Dot-prod is specialized on M/K, tile shape, and activation dtype.
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CompileKey:
|
||||
m: int
|
||||
k: int
|
||||
bs: int
|
||||
a_dtype: torch.dtype
|
||||
token_groups: int = 1
|
||||
epb: int = 1
|
||||
|
||||
@staticmethod
|
||||
def kernel(compile_key: CompileKey) -> Any:
|
||||
from ._ll_fp32w_dotprod import LLFp32WDotprod
|
||||
|
||||
return LLFp32WDotprod(
|
||||
m=compile_key.m,
|
||||
k=compile_key.k,
|
||||
bs=compile_key.bs,
|
||||
token_groups=compile_key.token_groups,
|
||||
epb=compile_key.epb,
|
||||
use_pdl=use_pdl(),
|
||||
)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
M: int,
|
||||
K: int,
|
||||
N: int,
|
||||
a_dtype: torch.dtype,
|
||||
) -> CompileKey:
|
||||
default_config = (
|
||||
_DEFAULT_DOTPROD_GROUPED_CONFIG
|
||||
if N <= 128 and M >= 12 and M % 2 == 0
|
||||
else _DEFAULT_DOTPROD_CONFIG
|
||||
)
|
||||
tuned_configs = _TUNED_DOTPROD_CONFIGS.get((K, N))
|
||||
config = (
|
||||
tuned_configs.get(M, default_config) if tuned_configs else default_config
|
||||
)
|
||||
return self.CompileKey(
|
||||
m=M,
|
||||
k=K,
|
||||
bs=config[0],
|
||||
a_dtype=a_dtype,
|
||||
token_groups=config[1],
|
||||
epb=config[2],
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
*,
|
||||
shapes: Iterable[tuple[int, int]],
|
||||
m_values: Iterable[int],
|
||||
a_dtypes: Iterable[torch.dtype] = (torch.bfloat16,),
|
||||
) -> list[CompileKey]:
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
zip_inputs(*(dict(K=K, N=N) for K, N in shapes)),
|
||||
M=tuple(m_values),
|
||||
a_dtype=tuple(a_dtypes),
|
||||
)
|
||||
|
||||
def compile(self, compile_key: CompileKey) -> None:
|
||||
if compile_key in self._compiled_cache:
|
||||
return
|
||||
if compile_key.token_groups not in (1, 2):
|
||||
raise ValueError("ll_fp32w_gemm supports token_groups 1 or 2")
|
||||
if compile_key.epb not in (1, 2):
|
||||
raise ValueError("ll_fp32w_gemm supports epb 1 or 2")
|
||||
if compile_key.m % compile_key.token_groups != 0:
|
||||
raise ValueError("M must be divisible by token_groups")
|
||||
|
||||
cute, _ = cute_context()
|
||||
N = cute.sym_int()
|
||||
stride_divisibility = math.gcd(8, compile_key.k)
|
||||
hidden_states, router_weight, output = make_fake_gemm_tensors(
|
||||
M=compile_key.m,
|
||||
K=compile_key.k,
|
||||
N=N,
|
||||
a_dtype=compile_key.a_dtype,
|
||||
b_dtype=torch.float32,
|
||||
divisibility=stride_divisibility,
|
||||
)
|
||||
compiled = cute.compile(
|
||||
self.kernel(compile_key),
|
||||
hidden_states,
|
||||
router_weight,
|
||||
output,
|
||||
1, # runtime N placeholder for fake-tensor compile
|
||||
current_cuda_stream(),
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
self._compiled_cache[compile_key] = compiled
|
||||
|
||||
@staticmethod
|
||||
def _validate_inputs(
|
||||
hidden_states: torch.Tensor,
|
||||
router_weight: torch.Tensor,
|
||||
output_dtype: torch.dtype,
|
||||
) -> None:
|
||||
if hidden_states.dtype not in _SUPPORTED_ACTIVATION_DTYPES:
|
||||
raise ValueError("hidden_states must have dtype bf16, fp16, or fp32")
|
||||
if router_weight.dtype != torch.float32:
|
||||
raise ValueError("router_weight must have dtype=float32")
|
||||
validate_common_gemm_inputs(
|
||||
hidden_states,
|
||||
router_weight,
|
||||
output_dtype,
|
||||
op_name="ll_fp32w_gemm",
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
hidden_states: torch.Tensor, # [M, K] bf16/fp16/fp32
|
||||
router_weight: torch.Tensor, # [N, K] fp32
|
||||
output_dtype: torch.dtype = torch.float32,
|
||||
) -> torch.Tensor: # [M, N] fp32
|
||||
self._validate_inputs(hidden_states, router_weight, output_dtype)
|
||||
|
||||
M, K = hidden_states.shape
|
||||
N = router_weight.shape[0]
|
||||
compile_key = self.dispatch(M=M, K=K, N=N, a_dtype=hidden_states.dtype)
|
||||
compiled = self._get_or_compile(
|
||||
compile_key,
|
||||
runtime_context={"M": M, "K": K, "N": N},
|
||||
)
|
||||
|
||||
stream = current_cuda_stream()
|
||||
output = torch.empty(M, N, dtype=output_dtype, device=hidden_states.device)
|
||||
compiled(hidden_states, router_weight, output, N, stream)
|
||||
return output
|
||||
|
||||
|
||||
LL_FP32W_GEMM_KERNEL = LLFp32WGemm()
|
||||
Reference in New Issue
Block a user