[Perf][GLM-5.2] Blackwell decode optimizations (#48597)

Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com>
Signed-off-by: zhou <zhou@zhoudeMacBook-Pro.local>
Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: zhou <zhou@zhoudeMacBook-Pro.local>
Co-authored-by: stefankoncarevic <Stefan.Koncarevic@amd.com>
This commit is contained in:
xiaozhoupy
2026-07-23 21:36:27 -07:00
committed by GitHub
co-authored by Claude Giancarlo Delfin Thien Tran OpenAI Codex zhou stefankoncarevic
parent 0d77325b10
commit dd72658e7d
29 changed files with 2139 additions and 138 deletions
+262
View File
@@ -0,0 +1,262 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//
// Skinny GEMM: activation(bf16) x weight(bf16)^T -> bf16, for decode-time
// M <= 32 with a large reduction dim. Replaces cuBLAS splitK (GEMM +
// splitKreduce) with a single block-per-output-column kernel; these shapes
// are weight-bandwidth-bound, so one coalesced pass over the weight at
// fp32 accumulation is optimal. Adapted from fp32_router_gemm.cu.
//
// First user: the DeepSeek-V32/GLM-5.2 MTP eh_proj (K=2*hidden=12288,
// N=hidden/TP), whose cuBLAS splitK pick costs ~34.6us vs the ~19us
// bandwidth floor per replicated read (and ~4us once column-parallel).
#include <cuda_bf16.h>
#include <cuda_runtime.h>
// ---------------------------------------------------------------------------
// Load helpers (8 x bf16 = one uint4 load, converted to fp32)
// ---------------------------------------------------------------------------
namespace skinny {
constexpr int VPT = 8; // bf16 values per thread per load
__device__ __forceinline__ void load_bf16x8(__nv_bfloat16 const* ptr,
float* dst) {
uint4 v = *reinterpret_cast<uint4 const*>(ptr);
__nv_bfloat16 const* p = reinterpret_cast<__nv_bfloat16 const*>(&v);
#pragma unroll
for (int i = 0; i < VPT; i++) dst[i] = __bfloat162float(p[i]);
}
// Streaming variant for the weight: each row is read exactly once across the
// whole grid, so bypass L2 residency (evict-first). Measured -1.4us at M=1.
__device__ __forceinline__ void load_bf16x8_cs(__nv_bfloat16 const* ptr,
float* dst) {
uint4 v = __ldcs(reinterpret_cast<uint4 const*>(ptr));
__nv_bfloat16 const* p = reinterpret_cast<__nv_bfloat16 const*>(&v);
#pragma unroll
for (int i = 0; i < VPT; i++) dst[i] = __bfloat162float(p[i]);
}
// ---------------------------------------------------------------------------
// Kernel: each block computes kNPB output columns for all kNumTokens rows.
// grid = kN / kNPB, block = kBlockSize threads. K is reduced VPT elements
// per thread per iteration; fp32 accumulation, warp butterfly + smem
// finalize, bf16 store.
// ---------------------------------------------------------------------------
template <int kBlockSize, int kNumTokens, int kNPB, int kPF, int kN, int kK>
__global__ __launch_bounds__(kBlockSize, 1) void bf16_skinny_gemm_kernel(
__nv_bfloat16* out, __nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b,
int64_t out_stride) {
constexpr int k_elems_per_iter = VPT * kBlockSize;
constexpr int k_iterations = kK / k_elems_per_iter;
static_assert(kK % k_elems_per_iter == 0);
constexpr int kWarpSize = 32;
constexpr int kNumWarps = kBlockSize / kWarpSize;
int const n_base = blockIdx.x * kNPB;
int const tid = threadIdx.x;
int const warpId = tid / kWarpSize;
int const laneId = tid % kWarpSize;
float acc[kNumTokens][kNPB] = {};
__shared__ float sm_reduction[kNumTokens][kNPB][kNumWarps];
// Register prefetch (kPF > 0): W does not depend on the predecessor, so
// the first kPF iterations' weight chunks are loaded raw BEFORE the
// dependency sync; with a PDL-releasing producer (fused_eh_norm fires
// gdc_launch_dependents early) these DRAM round trips overlap the norm.
// Pair-measured on B300 (norm+gemm in one graph, full 6144x12288):
// M=1 pf2 28.43us vs pf0 29.00us; deeper prefetch or M >= 2 regresses
// (register pressure), hence the per-M selection in the launcher.
uint4 w_pre[kPF > 0 ? kPF : 1][kNPB];
#pragma unroll
for (int pf = 0; pf < kPF; pf++) {
#pragma unroll
for (int n = 0; n < kNPB; n++) {
w_pre[pf][n] =
*reinterpret_cast<uint4 const*>(mat_b + (size_t)(n_base + n) * kK +
pf * k_elems_per_iter + tid * VPT);
}
}
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
cudaGridDependencySynchronize();
#endif
#pragma unroll
for (int ki = 0; ki < k_iterations; ki++) {
int const k_base = ki * k_elems_per_iter + tid * VPT;
float b_float[kNPB][VPT];
if (ki < kPF) {
#pragma unroll
for (int n = 0; n < kNPB; n++) {
__nv_bfloat16 const* p =
reinterpret_cast<__nv_bfloat16 const*>(&w_pre[ki][n]);
#pragma unroll
for (int v = 0; v < VPT; v++) b_float[n][v] = __bfloat162float(p[v]);
}
} else {
#pragma unroll
for (int n = 0; n < kNPB; n++) {
load_bf16x8_cs(mat_b + (size_t)(n_base + n) * kK + k_base, b_float[n]);
}
}
#pragma unroll
for (int m = 0; m < kNumTokens; m++) {
float a_float[VPT];
load_bf16x8(mat_a + (size_t)m * kK + k_base, a_float);
#pragma unroll
for (int n = 0; n < kNPB; n++) {
#pragma unroll
for (int k = 0; k < VPT; k++) {
acc[m][n] += a_float[k] * b_float[n][k];
}
}
}
}
// Warp-level butterfly reduction
#pragma unroll
for (int m = 0; m < kNumTokens; m++) {
#pragma unroll
for (int n = 0; n < kNPB; n++) {
float sum = acc[m][n];
sum += __shfl_xor_sync(0xffffffff, sum, 16);
sum += __shfl_xor_sync(0xffffffff, sum, 8);
sum += __shfl_xor_sync(0xffffffff, sum, 4);
sum += __shfl_xor_sync(0xffffffff, sum, 2);
sum += __shfl_xor_sync(0xffffffff, sum, 1);
if (laneId == 0) sm_reduction[m][n][warpId] = sum;
}
}
__syncthreads();
// Parallel finalize: one thread per (m, n) output.
for (int idx = tid; idx < kNumTokens * kNPB; idx += kBlockSize) {
int const m = idx / kNPB;
int const n = idx % kNPB;
float final_sum = 0.0f;
#pragma unroll
for (int w = 0; w < kNumWarps; w++) final_sum += sm_reduction[m][n][w];
out[(size_t)m * out_stride + n_base + n] = __float2bfloat16(final_sum);
}
// Trigger after our stores: harmless hardening, not a guarantee — the
// trigger only permits dependent-launch scheduling and carries no memory
// visibility semantics, so a PDL consumer must still gridsync before
// reading our output. Every current consumer is a plain launch (full
// stream order); firing late just avoids an unnecessarily early launch
// window and matches fp32_router_gemm.cu.
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
cudaTriggerProgrammaticLaunchCompletion();
#endif
}
} // namespace skinny
// ---------------------------------------------------------------------------
// Launcher
// ---------------------------------------------------------------------------
template <int kBlockSize, int kNPB, int kNumTokens, int kN, int kK>
void invokeBf16SkinnyGemm(__nv_bfloat16* output, __nv_bfloat16 const* mat_a,
__nv_bfloat16 const* mat_b, int64_t out_stride,
cudaStream_t stream) {
static_assert(kN % kNPB == 0);
// Weight prefetch depth: only M=1 measured a win (see kernel comment).
constexpr int kPF = (kNumTokens == 1) ? 2 : 0;
cudaLaunchConfig_t config;
config.gridDim = kN / kNPB;
config.blockDim = kBlockSize;
config.dynamicSmemBytes = 0;
config.stream = stream;
cudaLaunchAttribute attrs[1];
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
attrs[0].val.programmaticStreamSerializationAllowed = 1;
config.numAttrs = 1;
config.attrs = attrs;
cudaLaunchKernelEx(&config,
skinny::bf16_skinny_gemm_kernel<kBlockSize, kNumTokens,
kNPB, kPF, kN, kK>,
output, mat_a, mat_b, out_stride);
}
// ---------------------------------------------------------------------------
// Explicit instantiations. M = 1..32; (N, K) pairs:
// (768, 12288) eh_proj shard, TP8
// (1536, 12288) eh_proj shard, TP4
// (6144, 12288) eh_proj unsharded
// kNPB (B300 sweep, M=1): 6144 -> 2 (3072 blocks, 23.2us vs 25.3 at kNPB=8;
// narrow blocks minimize wave quantization); shards 768/1536 keep 4.
// ---------------------------------------------------------------------------
#define INSTANTIATE(M, NPB, N, K) \
template void invokeBf16SkinnyGemm<128, NPB, M, N, K>( \
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int64_t, \
cudaStream_t);
#define INSTANTIATE_ALL_M(NPB, N, K) \
INSTANTIATE(1, NPB, N, K) \
INSTANTIATE(2, NPB, N, K) \
INSTANTIATE(3, NPB, N, K) \
INSTANTIATE(4, NPB, N, K) \
INSTANTIATE(5, NPB, N, K) \
INSTANTIATE(6, NPB, N, K) \
INSTANTIATE(7, NPB, N, K) \
INSTANTIATE(8, NPB, N, K) \
INSTANTIATE(9, NPB, N, K) \
INSTANTIATE(10, NPB, N, K) \
INSTANTIATE(11, NPB, N, K) \
INSTANTIATE(12, NPB, N, K) \
INSTANTIATE(13, NPB, N, K) \
INSTANTIATE(14, NPB, N, K) \
INSTANTIATE(15, NPB, N, K) \
INSTANTIATE(16, NPB, N, K) \
INSTANTIATE(17, NPB, N, K) \
INSTANTIATE(18, NPB, N, K) \
INSTANTIATE(19, NPB, N, K) \
INSTANTIATE(20, NPB, N, K) \
INSTANTIATE(21, NPB, N, K) \
INSTANTIATE(22, NPB, N, K) \
INSTANTIATE(23, NPB, N, K) \
INSTANTIATE(24, NPB, N, K) \
INSTANTIATE(25, NPB, N, K) \
INSTANTIATE(26, NPB, N, K) \
INSTANTIATE(27, NPB, N, K) \
INSTANTIATE(28, NPB, N, K) \
INSTANTIATE(29, NPB, N, K) \
INSTANTIATE(30, NPB, N, K) \
INSTANTIATE(31, NPB, N, K) \
INSTANTIATE(32, NPB, N, K)
INSTANTIATE_ALL_M(4, 768, 12288)
INSTANTIATE_ALL_M(4, 1536, 12288)
INSTANTIATE_ALL_M(2, 6144, 12288)
// LL-mode (M<=8 wiring guard) backbone shapes, B300 sweep vs cuBLAS:
// q_b_proj (2048, 2048): 1.67x/1.29x/1.15x at M=4/6/8 (NPB=4 within
// 0.1us of per-M best)
// shared-expert gate_up (512, 6144): 1.95x/1.58x/1.40x at M=4/6/8
// cuBLAS keeps qkv_a (2624,6144) and o_proj (6144,2048) — already at
// 3.4-3.8 TB/s there; the GEMV loses on activation re-reads.
INSTANTIATE_ALL_M(4, 2048, 2048)
INSTANTIATE_ALL_M(4, 512, 6144)
// fused_qkv_a (2624, 6144), 32MB: skinny wins ONLY at M<=2 (B300: M=1
// 6.99us vs cuBLAS 9.21 = 1.32x, M=2 1.24x; M>=4 cuBLAS holds at 3.5TB/s
// and every alternative loses — cublasLt top-8 3.6TB/s wall, DeepGEMM
// 0.71x, wmma+cp.async custom 0.30x pending a TMA rewrite).
INSTANTIATE_ALL_M(4, 2624, 6144)
// DSv3.2 (TP8) siblings of the GLM shapes above, same dual-chip matrix:
// fused_qkv_a (2112, 7168), 30MB: wins M<=2 (M=1 1.30-1.34x)
// MTP eh_proj (7168, 14336), 205MB: wins M<=2 (M=1 1.12-1.16x)
INSTANTIATE_ALL_M(4, 2112, 7168)
INSTANTIATE_ALL_M(2, 7168, 14336)
#undef INSTANTIATE_ALL_M
#undef INSTANTIATE
@@ -0,0 +1,170 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#include <torch/csrc/stable/library.h>
#include <torch/csrc/stable/tensor.h>
#include <torch/headeronly/core/ScalarType.h>
#include "core/registration.h"
#include "libtorch_stable/torch_utils.h"
#include <cuda_bf16.h>
#include <cuda_runtime.h>
#include <stdexcept>
namespace {
inline int getSMVersion() {
auto* props = get_device_prop();
return props->major * 10 + props->minor;
}
} // namespace
static constexpr int SKINNY_MAX_TOKENS = 32;
// Supported (N, K) pairs (must match the instantiations in
// bf16_skinny_gemm.cu): eh_proj shard TP8 / TP4 / unsharded.
static inline bool bf16_skinny_gemm_supported(int n, int k) {
if (k == 12288 && (n == 768 || n == 1536 || n == 6144)) return true;
// LL-mode backbone shapes (wire callers with an M <= 8 guard; the GEMV
// family loses to cuBLAS at larger M).
if (k == 2048 && n == 2048) return true; // q_b_proj (TP8)
if (k == 6144 && n == 2624) return true; // fused_qkv_a (wire M <= 2 only)
if (k == 7168 && n == 2112) return true; // DSv3.2 fused_qkv_a (M <= 2)
if (k == 14336 && n == 7168) return true; // DSv3.2 eh_proj (M <= 2)
if (k == 6144 && n == 512) return true; // shared-expert gate_up (TP8)
return false;
}
// Forward declarations - template params must match bf16_skinny_gemm.cu
template <int kBlockSize, int kNPB, int kNumTokens, int kN, int kK>
void invokeBf16SkinnyGemm(__nv_bfloat16* output, __nv_bfloat16 const* mat_a,
__nv_bfloat16 const* mat_b, int64_t out_stride,
cudaStream_t stream);
template <int kNPB, int kN, int kK, int kBegin, int kEnd>
struct SkinnyLoopUnroller {
static void unroll(int num_tokens, __nv_bfloat16* output,
__nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b,
int64_t out_stride, cudaStream_t stream) {
if (num_tokens == kBegin) {
invokeBf16SkinnyGemm<128, kNPB, kBegin, kN, kK>(output, mat_a, mat_b,
out_stride, stream);
} else {
SkinnyLoopUnroller<kNPB, kN, kK, kBegin + 1, kEnd>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
}
}
};
template <int kNPB, int kN, int kK, int kEnd>
struct SkinnyLoopUnroller<kNPB, kN, kK, kEnd, kEnd> {
static void unroll(int num_tokens, __nv_bfloat16* output,
__nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b,
int64_t out_stride, cudaStream_t stream) {
if (num_tokens == kEnd) {
invokeBf16SkinnyGemm<128, kNPB, kEnd, kN, kK>(output, mat_a, mat_b,
out_stride, stream);
} else {
throw std::invalid_argument(
"bf16_skinny_gemm: num_tokens must be in [1, 32]");
}
}
};
static void dispatchBf16SkinnyGemm(int n, int k, int num_tokens,
__nv_bfloat16* output,
__nv_bfloat16 const* mat_a,
__nv_bfloat16 const* mat_b,
int64_t out_stride, cudaStream_t stream) {
if (n == 768 && k == 12288) {
SkinnyLoopUnroller<4, 768, 12288, 1, SKINNY_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
} else if (n == 1536 && k == 12288) {
SkinnyLoopUnroller<4, 1536, 12288, 1, SKINNY_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
} else if (n == 6144 && k == 12288) {
SkinnyLoopUnroller<2, 6144, 12288, 1, SKINNY_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
} else if (n == 2048 && k == 2048) {
SkinnyLoopUnroller<4, 2048, 2048, 1, SKINNY_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
} else if (n == 2624 && k == 6144) {
SkinnyLoopUnroller<4, 2624, 6144, 1, SKINNY_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
} else if (n == 2112 && k == 7168) {
SkinnyLoopUnroller<4, 2112, 7168, 1, SKINNY_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
} else if (n == 7168 && k == 14336) {
SkinnyLoopUnroller<2, 7168, 14336, 1, SKINNY_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
} else if (n == 512 && k == 6144) {
SkinnyLoopUnroller<4, 512, 6144, 1, SKINNY_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
} else {
throw std::invalid_argument("bf16_skinny_gemm: unsupported (N, K) pair");
}
}
void bf16_skinny_gemm(
torch::stable::Tensor& output, // [num_tokens, N] bf16
torch::stable::Tensor const& mat_a, // [num_tokens, K] bf16
torch::stable::Tensor const& mat_b // [N, K] bf16
) {
STD_TORCH_CHECK(output.dim() == 2 && mat_a.dim() == 2 && mat_b.dim() == 2);
STD_TORCH_CHECK(output.is_cuda() && mat_a.is_cuda() && mat_b.is_cuda(),
"bf16_skinny_gemm: all tensors must be CUDA tensors");
STD_TORCH_CHECK(output.get_device_index() == mat_a.get_device_index() &&
output.get_device_index() == mat_b.get_device_index(),
"bf16_skinny_gemm: all tensors must be on the same device");
STD_TORCH_CHECK(mat_a.is_contiguous() && mat_b.is_contiguous(),
"bf16_skinny_gemm: inputs must be contiguous");
// Output may be a column-slice view of a wider padded buffer: unit column
// stride, row stride >= N (rows must not overlap).
STD_TORCH_CHECK(output.stride(1) == 1,
"bf16_skinny_gemm: output columns must be contiguous");
const int num_tokens = mat_a.size(0);
const int n = mat_b.size(0);
const int k = mat_a.size(1);
STD_TORCH_CHECK(output.size(0) == num_tokens && output.size(1) == n,
"bf16_skinny_gemm: output must be [num_tokens, N]");
STD_TORCH_CHECK(mat_b.size(1) == k,
"bf16_skinny_gemm: mat_a and mat_b must share K");
STD_TORCH_CHECK(bf16_skinny_gemm_supported(n, k),
"bf16_skinny_gemm: unsupported (N, K) pair");
const int64_t out_stride = output.stride(0);
STD_TORCH_CHECK(num_tokens <= 1 || out_stride >= n,
"bf16_skinny_gemm: output rows overlap");
STD_TORCH_CHECK(num_tokens >= 0 && num_tokens <= SKINNY_MAX_TOKENS,
"bf16_skinny_gemm: num_tokens must be in [0, 32]");
STD_TORCH_CHECK(
mat_a.scalar_type() == torch::headeronly::ScalarType::BFloat16 &&
mat_b.scalar_type() == torch::headeronly::ScalarType::BFloat16 &&
output.scalar_type() == torch::headeronly::ScalarType::BFloat16,
"bf16_skinny_gemm: all tensors must be bfloat16");
// Empty batch (e.g. an empty rank at a DP/PP boundary): nothing to compute.
if (num_tokens == 0) {
return;
}
const torch::stable::accelerator::DeviceGuard device_guard(
mat_a.get_device_index());
STD_TORCH_CHECK(getSMVersion() >= 90, "bf16_skinny_gemm: requires SM90+");
auto stream = get_current_cuda_stream(mat_a.get_device_index());
dispatchBf16SkinnyGemm(
n, k, num_tokens,
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()),
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), out_stride,
stream);
}
STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) {
m.impl("bf16_skinny_gemm", TORCH_BOX(&bf16_skinny_gemm));
}
+112 -36
View File
@@ -391,7 +391,7 @@ struct MmaComputer {
static constexpr int n_iter_cnt =
(tile_n + 7) /
8; // Possible to have non-1 n_iter_cnt for ab_swap m16 case.
static_assert(m_iter_cnt == 1);
static_assert(m_iter_cnt == 1 || m_iter_cnt == 2);
static_assert(n_iter_cnt == 1 || n_iter_cnt == 2);
__device__ MmaComputer(bf16_t* gmem_c_local_, bf16_t* smem_a_,
@@ -416,13 +416,18 @@ struct MmaComputer {
public:
__device__ void prepare() {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
// Fragment addressing is per 16-row ldmatrix tile; m_iter selects the
// 16-row half within tile_m.
#pragma unroll
for (int i = 0; i < k_phase_cnt; i++) {
int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256;
int m_idx = linear_idx % tile_m;
int k_idx = linear_idx / tile_m + warp_k_offset_in_tile_k;
k_idx = apply_swizzle_343_on_elem_row_col<bf16_t>(m_idx, k_idx);
a_smem_offsets[0][i] = m_idx * tile_k + k_idx;
for (int m = 0; m < m_iter_cnt; m++) {
#pragma unroll
for (int i = 0; i < k_phase_cnt; i++) {
int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256;
int m_idx = linear_idx % 16 + m * 16;
int k_idx = linear_idx / 16 + warp_k_offset_in_tile_k;
k_idx = apply_swizzle_343_on_elem_row_col<bf16_t>(m_idx, k_idx);
a_smem_offsets[m][i] = m_idx * tile_k + k_idx;
}
}
#pragma unroll
for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) {
@@ -446,11 +451,14 @@ struct MmaComputer {
wait_barrier(smem_barrier + 0 + stage_idx * 2, phase_bit);
#pragma unroll
for (int i = 0; i < k_phase_cnt; i++) {
int smem_offset = a_smem_offsets[0][i];
bf16_t* smem_ptr_this_iter =
smem_a + stage_idx * tile_m * tile_k + smem_offset;
ldsm_x4(smem_ptr_this_iter, reinterpret_cast<uint32_t*>(a_reg[0][i]));
for (int m = 0; m < m_iter_cnt; m++) {
#pragma unroll
for (int i = 0; i < k_phase_cnt; i++) {
int smem_offset = a_smem_offsets[m][i];
bf16_t* smem_ptr_this_iter =
smem_a + stage_idx * tile_m * tile_k + smem_offset;
ldsm_x4(smem_ptr_this_iter, reinterpret_cast<uint32_t*>(a_reg[m][i]));
}
}
#pragma unroll
@@ -469,9 +477,12 @@ struct MmaComputer {
for (int k_iter_idx = 0; k_iter_idx < k_phase_cnt; k_iter_idx++) {
#pragma unroll
for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) {
hmma_16_8_16_f32acc_bf16ab(
acc_reg[0][n_iter_idx], a_reg[0][k_iter_idx],
b_reg[n_iter_idx][k_iter_idx], acc_reg[0][n_iter_idx]);
#pragma unroll
for (int m = 0; m < m_iter_cnt; m++) {
hmma_16_8_16_f32acc_bf16ab(
acc_reg[m][n_iter_idx], a_reg[m][k_iter_idx],
b_reg[n_iter_idx][k_iter_idx], acc_reg[m][n_iter_idx]);
}
}
}
::arrive_barrier(smem_barrier + 1 + stage_idx * 2);
@@ -486,14 +497,14 @@ struct MmaComputer {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
asm volatile("bar.sync %0, %1;" : : "r"(1), "r"(thread_cnt));
// reorganize the acc_reg
constexpr int thread_m = 2;
constexpr int thread_m = 2 * m_iter_cnt;
constexpr int thread_n = 2 * n_iter_cnt;
constexpr int cta_mma_n = n_iter_cnt * 8;
float acc_reg_reorg[thread_m][thread_n];
for (int i = 0; i < thread_m; i++) {
for (int j = 0; j < thread_n; j++) {
acc_reg_reorg[i][j] = acc_reg[0][j / 2][(j % 2) + (i * 2)];
acc_reg_reorg[i][j] = acc_reg[i / 2][j / 2][(j % 2) + (i % 2) * 2];
}
}
@@ -514,7 +525,8 @@ struct MmaComputer {
for (int m_idx_thread = 0; m_idx_thread < thread_m; m_idx_thread++) {
#pragma unroll
for (int n_idx_thread = 0; n_idx_thread < thread_n; n_idx_thread++) {
int m_idx = (lane_idx / 4) + m_idx_thread * 8;
int m_idx =
(lane_idx / 4) + (m_idx_thread % 2) * 8 + (m_idx_thread / 2) * 16;
int n_idx =
((lane_idx % 4) * 2) + (n_idx_thread % 2) + (n_idx_thread / 2) * 8;
smem_c[cosize_smem_c * warp_idx + smem_c_index_func(m_idx, n_idx)] =
@@ -587,7 +599,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
static_assert(
tile_k == 128 || tile_k == 256 || tile_k == 512 ||
tile_k == 1024); // tile_k must be larger than 64 since 4 warp splitK.
static_assert(tile_m == 16);
static_assert(tile_m == 16 || tile_m == 32);
constexpr int g2s_vec_bytes = 16;
constexpr int a_elem_bytes = 2;
constexpr int b_elem_bytes = 2;
@@ -647,7 +659,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
#endif
}
template <typename T, int kHdIn, int kHdOut, int kTileN>
template <typename T, int kHdIn, int kHdOut, int kTileN, int kTileM = 16>
void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
cudaStream_t const stream) {
constexpr int gemm_m = kHdOut; // 2112
@@ -655,7 +667,7 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
constexpr int gemm_k = kHdIn; // 7168
constexpr int batch_size = 1;
std::swap(mat_a, mat_b);
constexpr int tile_m = 16;
constexpr int tile_m = kTileM;
constexpr int tile_n = kTileN; // 8 or 16
constexpr int tile_k = std::max(256, 1024 / tile_n); // 256
constexpr int max_stage_cnt =
@@ -702,6 +714,44 @@ template void invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 16>(
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
cudaStream_t);
// GLM-5.2 fused_qkv_a (K=6144 -> N=2624). tile_m=32 so the grid is
// 2624/32 = 82 CTAs, a single wave on B300 (148 SMs); tile_m=16's 164 CTAs
// straddle two waves and drop to 2.9 TB/s vs 3.9-4.0 here. Beats cuBLAS
// at every decode M: 1.11-1.15x for M<=8 (tile_n=8), 1.12x at M=16
// (tile_n=16). The M<=2 dispatch still belongs to bf16_skinny_gemm
// (4.5 TB/s).
template void invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 8, 32>(
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
cudaStream_t);
template void invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 16, 32>(
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
cudaStream_t);
// GLM-5.2 q_b_proj TP8 (K=2048 -> N=2048). tile_m=16 keeps 128 CTAs (already
// a single wave) and measures ahead of tile_m=32 here. Beats cuBLAS
// 1.79-1.87x at M=3..8 (tile_n=8) and 1.19-1.30x at M=9..16 (tile_n=16);
// M<=2 belongs to bf16_skinny_gemm, M>=20 to cuBLAS.
template void invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 8>(
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
cudaStream_t);
template void invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 16>(
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
cudaStream_t);
// DSv3.2 q_b_proj TP8 (K=1536 -> N=3072). tile_m=32 keeps 96 CTAs (single
// wave; tile_m=16's 192 straddle two). Beats cuBLAS 1.6-1.8x at M=1..8 and
// 1.05-1.18x at M=9..16 on B300/B200; the whole 1..16 range dispatches here
// (no skinny tier: K=1536 does not fit the GEMV's 128x8 K-step).
template void invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 8, 32>(
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
cudaStream_t);
template void invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 16, 32>(
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
cudaStream_t);
void dsv3_fused_a_gemm(torch::stable::Tensor& output,
torch::stable::Tensor const& mat_a,
torch::stable::Tensor const& mat_b) {
@@ -710,12 +760,15 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output,
int const hd_in = mat_a.size(1);
int const hd_out = mat_b.size(1);
constexpr int kHdIn = 7168;
constexpr int kHdOut = 2112;
bool const is_dsv3 = hd_in == 7168 && hd_out == 2112;
bool const is_glm = hd_in == 6144 && hd_out == 2624;
bool const is_glm_qb = hd_in == 2048 && hd_out == 2048;
bool const is_ds_qb = hd_in == 1536 && hd_out == 3072;
STD_TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16,
"required 1 <= mat_a.shape[0] <= 16");
STD_TORCH_CHECK(hd_in == kHdIn, "required mat_a.shape[1] == 7168");
STD_TORCH_CHECK(hd_out == kHdOut, "required mat_b.shape[1] == 2112");
STD_TORCH_CHECK(is_dsv3 || is_glm || is_glm_qb || is_ds_qb,
"supported (hd_in, hd_out): (7168, 2112), (6144, 2624), "
"(2048, 2048), (1536, 3072)");
STD_TORCH_CHECK(output.size(0) == num_tokens,
"required output.shape[0] == mat_a.shape[0]");
STD_TORCH_CHECK(output.size(1) == hd_out,
@@ -738,18 +791,41 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output,
STD_TORCH_CHECK(getSMVersion() >= 90, "required CUDA ARCH >= SM_90");
auto stream = get_current_cuda_stream(mat_a.get_device_index());
if (num_tokens <= 8) {
invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 8>(
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()),
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), num_tokens,
stream);
auto* out = reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr());
auto* a = reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr());
auto* b = reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr());
if (is_dsv3) {
if (num_tokens <= 8) {
invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 8>(out, a, b, num_tokens,
stream);
} else {
invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 16>(out, a, b, num_tokens,
stream);
}
} else if (is_glm) {
if (num_tokens <= 8) {
invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 8, 32>(out, a, b, num_tokens,
stream);
} else {
invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 16, 32>(out, a, b, num_tokens,
stream);
}
} else if (is_glm_qb) {
if (num_tokens <= 8) {
invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 8>(out, a, b, num_tokens,
stream);
} else {
invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 16>(out, a, b, num_tokens,
stream);
}
} else {
invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 16>(
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()),
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), num_tokens,
stream);
if (num_tokens <= 8) {
invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 8, 32>(out, a, b, num_tokens,
stream);
} else {
invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 16, 32>(out, a, b, num_tokens,
stream);
}
}
}
@@ -49,6 +49,11 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD,
"Vec size is not matched.");
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
cudaGridDependencySynchronize();
cudaTriggerProgrammaticLaunchCompletion();
#endif
// Precompute SF layout parameter (constant for entire kernel).
int32_t const numKTiles = (outputCols + 63) / 64;
@@ -123,6 +128,11 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD,
"Vec size is not matched.");
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
cudaGridDependencySynchronize();
cudaTriggerProgrammaticLaunchCompletion();
#endif
int32_t const colIdx = blockDim.x * blockIdx.y + threadIdx.x;
int elem_idx = colIdx * CVT_FP4_ELTS_PER_THREAD;
@@ -201,6 +211,8 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
const torch::stable::accelerator::DeviceGuard device_guard(
input.get_device_index());
auto stream = get_current_cuda_stream(input.get_device_index());
auto* device_props = get_device_prop();
int const sm_version = device_props->major * 10 + device_props->minor;
int output_sf_n_unpadded = int(output_n / CVT_FP4_SF_VEC_SIZE);
@@ -224,10 +236,21 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
input.scalar_type(), "nvfp4_quant_kernel", [&] {
using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type;
auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
vllm::cvt_fp16_to_fp4<cuda_type, false><<<grid, block, 0, stream>>>(
m, n, output_n, num_padded_cols, input_ptr, input_sf_ptr,
reinterpret_cast<uint32_t*>(output_ptr),
reinterpret_cast<uint32_t*>(sf_out));
cudaLaunchConfig_t config = {};
config.gridDim = grid;
config.blockDim = block;
config.dynamicSmemBytes = 0;
config.stream = stream;
cudaLaunchAttribute attrs[1];
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
attrs[0].val.programmaticStreamSerializationAllowed = 1;
config.numAttrs = (sm_version >= 90) ? 1 : 0;
config.attrs = attrs;
cudaLaunchKernelEx(&config, vllm::cvt_fp16_to_fp4<cuda_type, false>,
m, n, output_n, num_padded_cols, input_ptr,
input_sf_ptr,
reinterpret_cast<uint32_t*>(output_ptr),
reinterpret_cast<uint32_t*>(sf_out));
});
} else {
int num_packed_cols = output_n / CVT_FP4_ELTS_PER_THREAD;
@@ -240,12 +263,21 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
input.scalar_type(), "nvfp4_quant_kernel", [&] {
using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type;
auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
vllm::cvt_fp16_to_fp4_sf_major<cuda_type, false>
<<<grid, block, 0, stream>>>(
m, n, output_n, output_sf_n_unpadded, num_packed_cols,
input_ptr, input_sf_ptr,
reinterpret_cast<uint32_t*>(output_ptr),
reinterpret_cast<uint32_t*>(sf_out));
cudaLaunchConfig_t config = {};
config.gridDim = grid;
config.blockDim = block;
config.dynamicSmemBytes = 0;
config.stream = stream;
cudaLaunchAttribute attrs[1];
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
attrs[0].val.programmaticStreamSerializationAllowed = 1;
config.numAttrs = (sm_version >= 90) ? 1 : 0;
config.attrs = attrs;
cudaLaunchKernelEx(
&config, vllm::cvt_fp16_to_fp4_sf_major<cuda_type, false>, m, n,
output_n, output_sf_n_unpadded, num_packed_cols, input_ptr,
input_sf_ptr, reinterpret_cast<uint32_t*>(output_ptr),
reinterpret_cast<uint32_t*>(sf_out));
});
}
}
+4
View File
@@ -330,6 +330,10 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
// conditionally compiled so impl registration is in source file
ops.def("fp32_router_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()");
// BF16 skinny GEMM (M<=32, weight-BW-bound shapes, e.g. MTP eh_proj).
// conditionally compiled so impl registration is in source file
ops.def("bf16_skinny_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()");
// reorder weight for AllSpark Ampere W8A16 Fused Gemm kernel
ops.def(
"rearrange_kn_weight_as_n32k16_order(Tensor b_qweight, Tensor b_scales, "