[Kernel] Extend CuTe DSL skinny GEMM to GLM-5.2 (#49791)

Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
xiaozhoupy
2026-08-03 16:49:41 -07:00
committed by GitHub
co-authored by OpenAI Codex Jee Jee Li
parent 6a9109d865
commit c810937573
7 changed files with 619 additions and 51 deletions
@@ -8,6 +8,8 @@ import cutlass.cute as cute
from cuda.bindings.driver import CUstream
from cutlass import const_expr
_PREFETCH_TILES = 2
class CuteSkinnyGemm:
"""Shape-dynamic low-latency GEMM for small token counts.
@@ -15,8 +17,9 @@ class CuteSkinnyGemm:
Computes ``C[M, N] = A[M, K] @ B[N, K].T + residual`` with BF16 or FP16
inputs, FP32 accumulators, and an output matching the input dtype. The
residual term is optional and is added before the output conversion. N and
K are runtime values. The tiny M dimension is fully unrolled alongside a
small set of tuning parameters.
K are runtime values unless the configuration specializes K at compile
time. The tiny M dimension is fully unrolled alongside a small set of
tuning parameters.
"""
def __init__(
@@ -30,9 +33,15 @@ class CuteSkinnyGemm:
k_unroll: int = 1,
has_residual: bool = False,
use_pdl: bool = False,
static_k: int | None = None,
) -> None:
if block_size % cute.arch.WARP_SIZE != 0:
raise ValueError("block_size must be a multiple of the warp size")
k_tile_size = block_size * vector_width
if static_k is not None and (
static_k % k_tile_size or static_k < _PREFETCH_TILES * k_tile_size
):
raise ValueError("static K must contain at least two complete tiles")
self.element_type = element_type
self.num_rows = num_rows
self.block_size = block_size
@@ -41,6 +50,7 @@ class CuteSkinnyGemm:
self.k_unroll = k_unroll
self.has_residual = has_residual
self.use_pdl = use_pdl
self.static_k = static_k
self.num_warps = block_size // cute.arch.WARP_SIZE
@cute.jit
@@ -66,7 +76,22 @@ class CuteSkinnyGemm:
num_bits_per_copy=self.vector_width * self.element_type.width,
load_cache_mode=cute.nvgpu.LoadCacheMode.STREAMING,
)
self.kernel(gA, gB, gResidual, gC, k, copy_a, copy_b).launch(
copy_b_prefetch = cute.make_copy_atom(
cute.nvgpu.CopyG2ROp(),
self.element_type,
num_bits_per_copy=self.vector_width * self.element_type.width,
load_cache_mode=cute.nvgpu.LoadCacheMode.ALWAYS,
)
self.kernel(
gA,
gB,
gResidual,
gC,
k,
copy_a,
copy_b,
copy_b_prefetch,
).launch(
grid=[cute.ceil_div(n, self.outputs_per_block), 1, 1],
block=[self.block_size, 1, 1],
smem=self.num_rows * self.outputs_per_block * self.num_warps * 4,
@@ -85,6 +110,7 @@ class CuteSkinnyGemm:
k_extent: cutlass.Int32,
copy_a: cute.CopyAtom,
copy_b: cute.CopyAtom,
copy_b_prefetch: cute.CopyAtom,
) -> None:
tidx, _, _ = cute.arch.thread_idx()
block_idx, _, _ = cute.arch.block_idx()
@@ -102,12 +128,8 @@ class CuteSkinnyGemm:
acc = cute.make_rmem_tensor(acc_layout, cutlass.Float32)
acc.fill(0.0)
if const_expr(self.use_pdl):
cute.arch.griddepcontrol_wait()
n_base = block_idx * outputs_per_block
k_tile_size: cutlass.Constexpr = block_size * vector_width
num_k_tiles = k_extent // k_tile_size
gA_vec = cute.logical_divide(gA, (None, vector_width))
gB_vec = cute.logical_divide(gB, (None, vector_width))
@@ -127,21 +149,72 @@ class CuteSkinnyGemm:
self.element_type,
)
for k_tile in cutlass.range(num_k_tiles, unroll=self.k_unroll):
for mi in cutlass.range_constexpr(num_rows):
cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None])
if const_expr(self.static_k is not None):
num_k_tiles: cutlass.Constexpr = self.static_k // k_tile_size
prefetched_b = cute.make_rmem_tensor(
cute.make_layout(
(_PREFETCH_TILES, outputs_per_block, vector_width),
stride=(outputs_per_block * vector_width, vector_width, 1),
),
self.element_type,
)
for k_tile in cutlass.range_constexpr(_PREFETCH_TILES):
for ni in cutlass.range_constexpr(outputs_per_block):
tB_static = tB_all[n_base + ni, (None, (tidx, None))]
cute.copy(
copy_b_prefetch,
tB_static[None, k_tile],
prefetched_b[k_tile, ni, None],
)
for ni in cutlass.range_constexpr(outputs_per_block):
n_idx = n_base + ni
tB = tB_all[n_idx, (None, (tidx, None))]
cute.copy(copy_b, tB[None, k_tile], b_regs[ni, None])
if const_expr(self.use_pdl):
cute.arch.griddepcontrol_wait()
for vi in cutlass.range_constexpr(vector_width):
for mi in cutlass.range_constexpr(num_rows):
for k_tile in cutlass.range_constexpr(num_k_tiles):
if const_expr(k_tile < _PREFETCH_TILES):
b_f32 = prefetched_b[k_tile, None, None].load().to(cutlass.Float32)
else:
for ni in cutlass.range_constexpr(outputs_per_block):
acc[mi, ni] = acc[mi, ni] + a_regs[mi, vi].to(
cutlass.Float32
) * b_regs[ni, vi].to(cutlass.Float32)
tB_static = tB_all[n_base + ni, (None, (tidx, None))]
cute.copy(
copy_b,
tB_static[None, k_tile],
b_regs[ni, None],
)
b_f32 = b_regs.load().to(cutlass.Float32)
for mi in cutlass.range_constexpr(num_rows):
cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None])
a_f32 = a_regs.load().to(cutlass.Float32)
for vi in cutlass.range_constexpr(vector_width):
for mi in cutlass.range_constexpr(num_rows):
for ni in cutlass.range_constexpr(outputs_per_block):
acc[mi, ni] += a_f32[mi, vi] * b_f32[ni, vi]
else:
if const_expr(self.use_pdl):
cute.arch.griddepcontrol_wait()
num_k_tiles = k_extent // k_tile_size
for k_tile in cutlass.range(num_k_tiles, unroll=self.k_unroll):
for mi in cutlass.range_constexpr(num_rows):
cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None])
for ni in cutlass.range_constexpr(outputs_per_block):
n_idx = n_base + ni
tB_dynamic = tB_all[n_idx, (None, (tidx, None))]
cute.copy(
copy_b,
tB_dynamic[None, k_tile],
b_regs[ni, None],
)
for vi in cutlass.range_constexpr(vector_width):
for mi in cutlass.range_constexpr(num_rows):
for ni in cutlass.range_constexpr(outputs_per_block):
acc[mi, ni] = acc[mi, ni] + a_regs[mi, vi].to(
cutlass.Float32
) * b_regs[ni, vi].to(cutlass.Float32)
for mi in cutlass.range_constexpr(num_rows):
for ni in cutlass.range_constexpr(outputs_per_block):
@@ -23,6 +23,7 @@ class SkinnyGemmConfig:
outputs_per_block: int
k_unroll: int = 1
vector_width: int = 8
static_k: int | None = None
class ShapeDynamicSkinnyGemm:
@@ -112,7 +113,11 @@ class ShapeDynamicSkinnyGemm:
element_type = self._cutlass_dtype(dtype)
n = cute.sym_int(divisibility=config.outputs_per_block)
k = cute.sym_int(divisibility=config.block_size * config.vector_width)
k = (
config.static_k
if config.static_k is not None
else cute.sym_int(divisibility=config.block_size * config.vector_width)
)
a = make_fake_tensor(
element_type,
(config.num_rows, k),
@@ -134,6 +139,7 @@ class ShapeDynamicSkinnyGemm:
k_unroll=config.k_unroll,
has_residual=has_residual,
use_pdl=self._use_pdl(),
static_k=config.static_k,
)
self._compiled[(dtype, config, has_residual)] = cute.compile(
kernel,
@@ -239,6 +245,8 @@ class ShapeDynamicSkinnyGemm:
raise ValueError(
"K must be divisible by block_size * vector_width for this config"
)
if config.static_k is not None and a.shape[1] != config.static_k:
raise ValueError("input K must match config static_k")
has_residual = residual is not None
cache_key = (a.dtype, config, has_residual)
if cache_key not in self._compiled: