[Kernel] Extend CuTe DSL skinny GEMM to GLM-5.2 (#49791)

Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
xiaozhoupy
2026-08-03 16:49:41 -07:00
committed by GitHub
co-authored by OpenAI Codex Jee Jee Li
parent 6a9109d865
commit c810937573
7 changed files with 619 additions and 51 deletions
+42 -24
View File
@@ -391,7 +391,7 @@ struct MmaComputer {
static constexpr int n_iter_cnt =
(tile_n + 7) /
8; // Possible to have non-1 n_iter_cnt for ab_swap m16 case.
static_assert(m_iter_cnt == 1);
static_assert(m_iter_cnt == 1 || m_iter_cnt == 2);
static_assert(n_iter_cnt == 1 || n_iter_cnt == 2);
__device__ MmaComputer(bf16_t* gmem_c_local_, bf16_t* smem_a_,
@@ -417,12 +417,15 @@ struct MmaComputer {
__device__ void prepare() {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
#pragma unroll
for (int i = 0; i < k_phase_cnt; i++) {
int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256;
int m_idx = linear_idx % tile_m;
int k_idx = linear_idx / tile_m + warp_k_offset_in_tile_k;
k_idx = apply_swizzle_343_on_elem_row_col<bf16_t>(m_idx, k_idx);
a_smem_offsets[0][i] = m_idx * tile_k + k_idx;
for (int m = 0; m < m_iter_cnt; m++) {
#pragma unroll
for (int i = 0; i < k_phase_cnt; i++) {
int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256;
int m_idx = linear_idx % 16 + m * 16;
int k_idx = linear_idx / 16 + warp_k_offset_in_tile_k;
k_idx = apply_swizzle_343_on_elem_row_col<bf16_t>(m_idx, k_idx);
a_smem_offsets[m][i] = m_idx * tile_k + k_idx;
}
}
#pragma unroll
for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) {
@@ -446,11 +449,14 @@ struct MmaComputer {
wait_barrier(smem_barrier + 0 + stage_idx * 2, phase_bit);
#pragma unroll
for (int i = 0; i < k_phase_cnt; i++) {
int smem_offset = a_smem_offsets[0][i];
bf16_t* smem_ptr_this_iter =
smem_a + stage_idx * tile_m * tile_k + smem_offset;
ldsm_x4(smem_ptr_this_iter, reinterpret_cast<uint32_t*>(a_reg[0][i]));
for (int m = 0; m < m_iter_cnt; m++) {
#pragma unroll
for (int i = 0; i < k_phase_cnt; i++) {
int smem_offset = a_smem_offsets[m][i];
bf16_t* smem_ptr_this_iter =
smem_a + stage_idx * tile_m * tile_k + smem_offset;
ldsm_x4(smem_ptr_this_iter, reinterpret_cast<uint32_t*>(a_reg[m][i]));
}
}
#pragma unroll
@@ -469,9 +475,12 @@ struct MmaComputer {
for (int k_iter_idx = 0; k_iter_idx < k_phase_cnt; k_iter_idx++) {
#pragma unroll
for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) {
hmma_16_8_16_f32acc_bf16ab(
acc_reg[0][n_iter_idx], a_reg[0][k_iter_idx],
b_reg[n_iter_idx][k_iter_idx], acc_reg[0][n_iter_idx]);
#pragma unroll
for (int m = 0; m < m_iter_cnt; m++) {
hmma_16_8_16_f32acc_bf16ab(
acc_reg[m][n_iter_idx], a_reg[m][k_iter_idx],
b_reg[n_iter_idx][k_iter_idx], acc_reg[m][n_iter_idx]);
}
}
}
::arrive_barrier(smem_barrier + 1 + stage_idx * 2);
@@ -486,14 +495,14 @@ struct MmaComputer {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
asm volatile("bar.sync %0, %1;" : : "r"(1), "r"(thread_cnt));
// reorganize the acc_reg
constexpr int thread_m = 2;
constexpr int thread_m = 2 * m_iter_cnt;
constexpr int thread_n = 2 * n_iter_cnt;
constexpr int cta_mma_n = n_iter_cnt * 8;
float acc_reg_reorg[thread_m][thread_n];
for (int i = 0; i < thread_m; i++) {
for (int j = 0; j < thread_n; j++) {
acc_reg_reorg[i][j] = acc_reg[0][j / 2][(j % 2) + (i * 2)];
acc_reg_reorg[i][j] = acc_reg[i / 2][j / 2][(j % 2) + (i % 2) * 2];
}
}
@@ -514,7 +523,8 @@ struct MmaComputer {
for (int m_idx_thread = 0; m_idx_thread < thread_m; m_idx_thread++) {
#pragma unroll
for (int n_idx_thread = 0; n_idx_thread < thread_n; n_idx_thread++) {
int m_idx = (lane_idx / 4) + m_idx_thread * 8;
int m_idx =
(lane_idx / 4) + (m_idx_thread % 2) * 8 + (m_idx_thread / 2) * 16;
int n_idx =
((lane_idx % 4) * 2) + (n_idx_thread % 2) + (n_idx_thread / 2) * 8;
smem_c[cosize_smem_c * warp_idx + smem_c_index_func(m_idx, n_idx)] =
@@ -587,7 +597,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
static_assert(
tile_k == 128 || tile_k == 256 || tile_k == 512 ||
tile_k == 1024); // tile_k must be larger than 64 since 4 warp splitK.
static_assert(tile_m == 16);
static_assert(tile_m == 16 || tile_m == 32);
constexpr int g2s_vec_bytes = 16;
constexpr int a_elem_bytes = 2;
constexpr int b_elem_bytes = 2;
@@ -647,7 +657,8 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
#endif
}
template <typename T, int kHdIn, int kHdOut, int kTileN, int kTileK = 256>
template <typename T, int kHdIn, int kHdOut, int kTileN, int kTileK = 256,
int kTileM = 16>
void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
cudaStream_t const stream, bool enable_pdl) {
constexpr int gemm_m = kHdOut;
@@ -655,7 +666,7 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
constexpr int gemm_k = kHdIn;
constexpr int batch_size = 1;
std::swap(mat_a, mat_b);
constexpr int tile_m = 16;
constexpr int tile_m = kTileM;
constexpr int tile_n = kTileN;
constexpr int tile_k = kTileK;
constexpr int max_stage_cnt =
@@ -695,15 +706,15 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
output, mat_a, mat_b, gemm_n);
}
template <typename T, int kHdIn, int kHdOut, int kTileK = 256>
template <typename T, int kHdIn, int kHdOut, int kTileK = 256, int kTileM = 16>
void invokeFusedAGemmForTokens(T* output, T const* mat_a, T const* mat_b,
int num_tokens, cudaStream_t const stream,
bool enable_pdl) {
if (num_tokens <= 8) {
invokeFusedAGemm<T, kHdIn, kHdOut, 8, kTileK>(
invokeFusedAGemm<T, kHdIn, kHdOut, 8, kTileK, kTileM>(
output, mat_a, mat_b, num_tokens, stream, enable_pdl);
} else {
invokeFusedAGemm<T, kHdIn, kHdOut, 16, kTileK>(
invokeFusedAGemm<T, kHdIn, kHdOut, 16, kTileK, kTileM>(
output, mat_a, mat_b, num_tokens, stream, enable_pdl);
}
}
@@ -782,6 +793,13 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output,
DISPATCH_DSV3_SHAPE(7168, 3216)
DISPATCH_DSV3_SHAPE(7168, 4224)
if (hd_in == 6144 && hd_out == 2624) {
invokeFusedAGemmForTokens<__nv_bfloat16, 6144, 2624, 256, 32>(
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl);
return;
}
DISPATCH_DSV3_SHAPE(2048, 2048)
#ifdef VLLM_K3_BENCH_SHAPES
// The selector routes these shapes to CuTe or the default GEMM, so they are
// never reached in production. They are compiled only for offline