[Kernel] Extend CuTe DSL skinny GEMM to GLM-5.2 (#49791)
Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Jee Jee Li
parent
6a9109d865
commit
c810937573
@@ -8,6 +8,8 @@ import cutlass.cute as cute
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import const_expr
|
||||
|
||||
_PREFETCH_TILES = 2
|
||||
|
||||
|
||||
class CuteSkinnyGemm:
|
||||
"""Shape-dynamic low-latency GEMM for small token counts.
|
||||
@@ -15,8 +17,9 @@ class CuteSkinnyGemm:
|
||||
Computes ``C[M, N] = A[M, K] @ B[N, K].T + residual`` with BF16 or FP16
|
||||
inputs, FP32 accumulators, and an output matching the input dtype. The
|
||||
residual term is optional and is added before the output conversion. N and
|
||||
K are runtime values. The tiny M dimension is fully unrolled alongside a
|
||||
small set of tuning parameters.
|
||||
K are runtime values unless the configuration specializes K at compile
|
||||
time. The tiny M dimension is fully unrolled alongside a small set of
|
||||
tuning parameters.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -30,9 +33,15 @@ class CuteSkinnyGemm:
|
||||
k_unroll: int = 1,
|
||||
has_residual: bool = False,
|
||||
use_pdl: bool = False,
|
||||
static_k: int | None = None,
|
||||
) -> None:
|
||||
if block_size % cute.arch.WARP_SIZE != 0:
|
||||
raise ValueError("block_size must be a multiple of the warp size")
|
||||
k_tile_size = block_size * vector_width
|
||||
if static_k is not None and (
|
||||
static_k % k_tile_size or static_k < _PREFETCH_TILES * k_tile_size
|
||||
):
|
||||
raise ValueError("static K must contain at least two complete tiles")
|
||||
self.element_type = element_type
|
||||
self.num_rows = num_rows
|
||||
self.block_size = block_size
|
||||
@@ -41,6 +50,7 @@ class CuteSkinnyGemm:
|
||||
self.k_unroll = k_unroll
|
||||
self.has_residual = has_residual
|
||||
self.use_pdl = use_pdl
|
||||
self.static_k = static_k
|
||||
self.num_warps = block_size // cute.arch.WARP_SIZE
|
||||
|
||||
@cute.jit
|
||||
@@ -66,7 +76,22 @@ class CuteSkinnyGemm:
|
||||
num_bits_per_copy=self.vector_width * self.element_type.width,
|
||||
load_cache_mode=cute.nvgpu.LoadCacheMode.STREAMING,
|
||||
)
|
||||
self.kernel(gA, gB, gResidual, gC, k, copy_a, copy_b).launch(
|
||||
copy_b_prefetch = cute.make_copy_atom(
|
||||
cute.nvgpu.CopyG2ROp(),
|
||||
self.element_type,
|
||||
num_bits_per_copy=self.vector_width * self.element_type.width,
|
||||
load_cache_mode=cute.nvgpu.LoadCacheMode.ALWAYS,
|
||||
)
|
||||
self.kernel(
|
||||
gA,
|
||||
gB,
|
||||
gResidual,
|
||||
gC,
|
||||
k,
|
||||
copy_a,
|
||||
copy_b,
|
||||
copy_b_prefetch,
|
||||
).launch(
|
||||
grid=[cute.ceil_div(n, self.outputs_per_block), 1, 1],
|
||||
block=[self.block_size, 1, 1],
|
||||
smem=self.num_rows * self.outputs_per_block * self.num_warps * 4,
|
||||
@@ -85,6 +110,7 @@ class CuteSkinnyGemm:
|
||||
k_extent: cutlass.Int32,
|
||||
copy_a: cute.CopyAtom,
|
||||
copy_b: cute.CopyAtom,
|
||||
copy_b_prefetch: cute.CopyAtom,
|
||||
) -> None:
|
||||
tidx, _, _ = cute.arch.thread_idx()
|
||||
block_idx, _, _ = cute.arch.block_idx()
|
||||
@@ -102,12 +128,8 @@ class CuteSkinnyGemm:
|
||||
acc = cute.make_rmem_tensor(acc_layout, cutlass.Float32)
|
||||
acc.fill(0.0)
|
||||
|
||||
if const_expr(self.use_pdl):
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
n_base = block_idx * outputs_per_block
|
||||
k_tile_size: cutlass.Constexpr = block_size * vector_width
|
||||
num_k_tiles = k_extent // k_tile_size
|
||||
|
||||
gA_vec = cute.logical_divide(gA, (None, vector_width))
|
||||
gB_vec = cute.logical_divide(gB, (None, vector_width))
|
||||
@@ -127,21 +149,72 @@ class CuteSkinnyGemm:
|
||||
self.element_type,
|
||||
)
|
||||
|
||||
for k_tile in cutlass.range(num_k_tiles, unroll=self.k_unroll):
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None])
|
||||
if const_expr(self.static_k is not None):
|
||||
num_k_tiles: cutlass.Constexpr = self.static_k // k_tile_size
|
||||
prefetched_b = cute.make_rmem_tensor(
|
||||
cute.make_layout(
|
||||
(_PREFETCH_TILES, outputs_per_block, vector_width),
|
||||
stride=(outputs_per_block * vector_width, vector_width, 1),
|
||||
),
|
||||
self.element_type,
|
||||
)
|
||||
for k_tile in cutlass.range_constexpr(_PREFETCH_TILES):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
tB_static = tB_all[n_base + ni, (None, (tidx, None))]
|
||||
cute.copy(
|
||||
copy_b_prefetch,
|
||||
tB_static[None, k_tile],
|
||||
prefetched_b[k_tile, ni, None],
|
||||
)
|
||||
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
n_idx = n_base + ni
|
||||
tB = tB_all[n_idx, (None, (tidx, None))]
|
||||
cute.copy(copy_b, tB[None, k_tile], b_regs[ni, None])
|
||||
if const_expr(self.use_pdl):
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
for vi in cutlass.range_constexpr(vector_width):
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
for k_tile in cutlass.range_constexpr(num_k_tiles):
|
||||
if const_expr(k_tile < _PREFETCH_TILES):
|
||||
b_f32 = prefetched_b[k_tile, None, None].load().to(cutlass.Float32)
|
||||
else:
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
acc[mi, ni] = acc[mi, ni] + a_regs[mi, vi].to(
|
||||
cutlass.Float32
|
||||
) * b_regs[ni, vi].to(cutlass.Float32)
|
||||
tB_static = tB_all[n_base + ni, (None, (tidx, None))]
|
||||
cute.copy(
|
||||
copy_b,
|
||||
tB_static[None, k_tile],
|
||||
b_regs[ni, None],
|
||||
)
|
||||
b_f32 = b_regs.load().to(cutlass.Float32)
|
||||
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None])
|
||||
a_f32 = a_regs.load().to(cutlass.Float32)
|
||||
|
||||
for vi in cutlass.range_constexpr(vector_width):
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
acc[mi, ni] += a_f32[mi, vi] * b_f32[ni, vi]
|
||||
else:
|
||||
if const_expr(self.use_pdl):
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
num_k_tiles = k_extent // k_tile_size
|
||||
for k_tile in cutlass.range(num_k_tiles, unroll=self.k_unroll):
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None])
|
||||
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
n_idx = n_base + ni
|
||||
tB_dynamic = tB_all[n_idx, (None, (tidx, None))]
|
||||
cute.copy(
|
||||
copy_b,
|
||||
tB_dynamic[None, k_tile],
|
||||
b_regs[ni, None],
|
||||
)
|
||||
|
||||
for vi in cutlass.range_constexpr(vector_width):
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
acc[mi, ni] = acc[mi, ni] + a_regs[mi, vi].to(
|
||||
cutlass.Float32
|
||||
) * b_regs[ni, vi].to(cutlass.Float32)
|
||||
|
||||
for mi in cutlass.range_constexpr(num_rows):
|
||||
for ni in cutlass.range_constexpr(outputs_per_block):
|
||||
|
||||
@@ -23,6 +23,7 @@ class SkinnyGemmConfig:
|
||||
outputs_per_block: int
|
||||
k_unroll: int = 1
|
||||
vector_width: int = 8
|
||||
static_k: int | None = None
|
||||
|
||||
|
||||
class ShapeDynamicSkinnyGemm:
|
||||
@@ -112,7 +113,11 @@ class ShapeDynamicSkinnyGemm:
|
||||
|
||||
element_type = self._cutlass_dtype(dtype)
|
||||
n = cute.sym_int(divisibility=config.outputs_per_block)
|
||||
k = cute.sym_int(divisibility=config.block_size * config.vector_width)
|
||||
k = (
|
||||
config.static_k
|
||||
if config.static_k is not None
|
||||
else cute.sym_int(divisibility=config.block_size * config.vector_width)
|
||||
)
|
||||
a = make_fake_tensor(
|
||||
element_type,
|
||||
(config.num_rows, k),
|
||||
@@ -134,6 +139,7 @@ class ShapeDynamicSkinnyGemm:
|
||||
k_unroll=config.k_unroll,
|
||||
has_residual=has_residual,
|
||||
use_pdl=self._use_pdl(),
|
||||
static_k=config.static_k,
|
||||
)
|
||||
self._compiled[(dtype, config, has_residual)] = cute.compile(
|
||||
kernel,
|
||||
@@ -239,6 +245,8 @@ class ShapeDynamicSkinnyGemm:
|
||||
raise ValueError(
|
||||
"K must be divisible by block_size * vector_width for this config"
|
||||
)
|
||||
if config.static_k is not None and a.shape[1] != config.static_k:
|
||||
raise ValueError("input K must match config static_k")
|
||||
has_residual = residual is not None
|
||||
cache_key = (a.dtype, config, has_residual)
|
||||
if cache_key not in self._compiled:
|
||||
|
||||
Reference in New Issue
Block a user