[Perf][GLM-5.2] Blackwell decode optimizations (#48597)
Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com> Signed-off-by: zhou <zhou@zhoudeMacBook-Pro.local> Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: zhou <zhou@zhoudeMacBook-Pro.local> Co-authored-by: stefankoncarevic <Stefan.Koncarevic@amd.com>
This commit is contained in:
co-authored by
Claude
Giancarlo Delfin
Thien Tran
OpenAI Codex
zhou
stefankoncarevic
parent
0d77325b10
commit
dd72658e7d
@@ -0,0 +1,262 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
//
|
||||
// Skinny GEMM: activation(bf16) x weight(bf16)^T -> bf16, for decode-time
|
||||
// M <= 32 with a large reduction dim. Replaces cuBLAS splitK (GEMM +
|
||||
// splitKreduce) with a single block-per-output-column kernel; these shapes
|
||||
// are weight-bandwidth-bound, so one coalesced pass over the weight at
|
||||
// fp32 accumulation is optimal. Adapted from fp32_router_gemm.cu.
|
||||
//
|
||||
// First user: the DeepSeek-V32/GLM-5.2 MTP eh_proj (K=2*hidden=12288,
|
||||
// N=hidden/TP), whose cuBLAS splitK pick costs ~34.6us vs the ~19us
|
||||
// bandwidth floor per replicated read (and ~4us once column-parallel).
|
||||
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Load helpers (8 x bf16 = one uint4 load, converted to fp32)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
namespace skinny {
|
||||
|
||||
constexpr int VPT = 8; // bf16 values per thread per load
|
||||
|
||||
__device__ __forceinline__ void load_bf16x8(__nv_bfloat16 const* ptr,
|
||||
float* dst) {
|
||||
uint4 v = *reinterpret_cast<uint4 const*>(ptr);
|
||||
__nv_bfloat16 const* p = reinterpret_cast<__nv_bfloat16 const*>(&v);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < VPT; i++) dst[i] = __bfloat162float(p[i]);
|
||||
}
|
||||
|
||||
// Streaming variant for the weight: each row is read exactly once across the
|
||||
// whole grid, so bypass L2 residency (evict-first). Measured -1.4us at M=1.
|
||||
__device__ __forceinline__ void load_bf16x8_cs(__nv_bfloat16 const* ptr,
|
||||
float* dst) {
|
||||
uint4 v = __ldcs(reinterpret_cast<uint4 const*>(ptr));
|
||||
__nv_bfloat16 const* p = reinterpret_cast<__nv_bfloat16 const*>(&v);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < VPT; i++) dst[i] = __bfloat162float(p[i]);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Kernel: each block computes kNPB output columns for all kNumTokens rows.
|
||||
// grid = kN / kNPB, block = kBlockSize threads. K is reduced VPT elements
|
||||
// per thread per iteration; fp32 accumulation, warp butterfly + smem
|
||||
// finalize, bf16 store.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
template <int kBlockSize, int kNumTokens, int kNPB, int kPF, int kN, int kK>
|
||||
__global__ __launch_bounds__(kBlockSize, 1) void bf16_skinny_gemm_kernel(
|
||||
__nv_bfloat16* out, __nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b,
|
||||
int64_t out_stride) {
|
||||
constexpr int k_elems_per_iter = VPT * kBlockSize;
|
||||
constexpr int k_iterations = kK / k_elems_per_iter;
|
||||
static_assert(kK % k_elems_per_iter == 0);
|
||||
constexpr int kWarpSize = 32;
|
||||
constexpr int kNumWarps = kBlockSize / kWarpSize;
|
||||
|
||||
int const n_base = blockIdx.x * kNPB;
|
||||
int const tid = threadIdx.x;
|
||||
int const warpId = tid / kWarpSize;
|
||||
int const laneId = tid % kWarpSize;
|
||||
|
||||
float acc[kNumTokens][kNPB] = {};
|
||||
__shared__ float sm_reduction[kNumTokens][kNPB][kNumWarps];
|
||||
|
||||
// Register prefetch (kPF > 0): W does not depend on the predecessor, so
|
||||
// the first kPF iterations' weight chunks are loaded raw BEFORE the
|
||||
// dependency sync; with a PDL-releasing producer (fused_eh_norm fires
|
||||
// gdc_launch_dependents early) these DRAM round trips overlap the norm.
|
||||
// Pair-measured on B300 (norm+gemm in one graph, full 6144x12288):
|
||||
// M=1 pf2 28.43us vs pf0 29.00us; deeper prefetch or M >= 2 regresses
|
||||
// (register pressure), hence the per-M selection in the launcher.
|
||||
uint4 w_pre[kPF > 0 ? kPF : 1][kNPB];
|
||||
#pragma unroll
|
||||
for (int pf = 0; pf < kPF; pf++) {
|
||||
#pragma unroll
|
||||
for (int n = 0; n < kNPB; n++) {
|
||||
w_pre[pf][n] =
|
||||
*reinterpret_cast<uint4 const*>(mat_b + (size_t)(n_base + n) * kK +
|
||||
pf * k_elems_per_iter + tid * VPT);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
cudaGridDependencySynchronize();
|
||||
#endif
|
||||
|
||||
#pragma unroll
|
||||
for (int ki = 0; ki < k_iterations; ki++) {
|
||||
int const k_base = ki * k_elems_per_iter + tid * VPT;
|
||||
|
||||
float b_float[kNPB][VPT];
|
||||
if (ki < kPF) {
|
||||
#pragma unroll
|
||||
for (int n = 0; n < kNPB; n++) {
|
||||
__nv_bfloat16 const* p =
|
||||
reinterpret_cast<__nv_bfloat16 const*>(&w_pre[ki][n]);
|
||||
#pragma unroll
|
||||
for (int v = 0; v < VPT; v++) b_float[n][v] = __bfloat162float(p[v]);
|
||||
}
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int n = 0; n < kNPB; n++) {
|
||||
load_bf16x8_cs(mat_b + (size_t)(n_base + n) * kK + k_base, b_float[n]);
|
||||
}
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int m = 0; m < kNumTokens; m++) {
|
||||
float a_float[VPT];
|
||||
load_bf16x8(mat_a + (size_t)m * kK + k_base, a_float);
|
||||
#pragma unroll
|
||||
for (int n = 0; n < kNPB; n++) {
|
||||
#pragma unroll
|
||||
for (int k = 0; k < VPT; k++) {
|
||||
acc[m][n] += a_float[k] * b_float[n][k];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Warp-level butterfly reduction
|
||||
#pragma unroll
|
||||
for (int m = 0; m < kNumTokens; m++) {
|
||||
#pragma unroll
|
||||
for (int n = 0; n < kNPB; n++) {
|
||||
float sum = acc[m][n];
|
||||
sum += __shfl_xor_sync(0xffffffff, sum, 16);
|
||||
sum += __shfl_xor_sync(0xffffffff, sum, 8);
|
||||
sum += __shfl_xor_sync(0xffffffff, sum, 4);
|
||||
sum += __shfl_xor_sync(0xffffffff, sum, 2);
|
||||
sum += __shfl_xor_sync(0xffffffff, sum, 1);
|
||||
if (laneId == 0) sm_reduction[m][n][warpId] = sum;
|
||||
}
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
// Parallel finalize: one thread per (m, n) output.
|
||||
for (int idx = tid; idx < kNumTokens * kNPB; idx += kBlockSize) {
|
||||
int const m = idx / kNPB;
|
||||
int const n = idx % kNPB;
|
||||
float final_sum = 0.0f;
|
||||
#pragma unroll
|
||||
for (int w = 0; w < kNumWarps; w++) final_sum += sm_reduction[m][n][w];
|
||||
out[(size_t)m * out_stride + n_base + n] = __float2bfloat16(final_sum);
|
||||
}
|
||||
|
||||
// Trigger after our stores: harmless hardening, not a guarantee — the
|
||||
// trigger only permits dependent-launch scheduling and carries no memory
|
||||
// visibility semantics, so a PDL consumer must still gridsync before
|
||||
// reading our output. Every current consumer is a plain launch (full
|
||||
// stream order); firing late just avoids an unnecessarily early launch
|
||||
// window and matches fp32_router_gemm.cu.
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace skinny
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Launcher
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
template <int kBlockSize, int kNPB, int kNumTokens, int kN, int kK>
|
||||
void invokeBf16SkinnyGemm(__nv_bfloat16* output, __nv_bfloat16 const* mat_a,
|
||||
__nv_bfloat16 const* mat_b, int64_t out_stride,
|
||||
cudaStream_t stream) {
|
||||
static_assert(kN % kNPB == 0);
|
||||
// Weight prefetch depth: only M=1 measured a win (see kernel comment).
|
||||
constexpr int kPF = (kNumTokens == 1) ? 2 : 0;
|
||||
cudaLaunchConfig_t config;
|
||||
config.gridDim = kN / kNPB;
|
||||
config.blockDim = kBlockSize;
|
||||
config.dynamicSmemBytes = 0;
|
||||
config.stream = stream;
|
||||
cudaLaunchAttribute attrs[1];
|
||||
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
|
||||
attrs[0].val.programmaticStreamSerializationAllowed = 1;
|
||||
config.numAttrs = 1;
|
||||
config.attrs = attrs;
|
||||
cudaLaunchKernelEx(&config,
|
||||
skinny::bf16_skinny_gemm_kernel<kBlockSize, kNumTokens,
|
||||
kNPB, kPF, kN, kK>,
|
||||
output, mat_a, mat_b, out_stride);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Explicit instantiations. M = 1..32; (N, K) pairs:
|
||||
// (768, 12288) eh_proj shard, TP8
|
||||
// (1536, 12288) eh_proj shard, TP4
|
||||
// (6144, 12288) eh_proj unsharded
|
||||
// kNPB (B300 sweep, M=1): 6144 -> 2 (3072 blocks, 23.2us vs 25.3 at kNPB=8;
|
||||
// narrow blocks minimize wave quantization); shards 768/1536 keep 4.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#define INSTANTIATE(M, NPB, N, K) \
|
||||
template void invokeBf16SkinnyGemm<128, NPB, M, N, K>( \
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int64_t, \
|
||||
cudaStream_t);
|
||||
|
||||
#define INSTANTIATE_ALL_M(NPB, N, K) \
|
||||
INSTANTIATE(1, NPB, N, K) \
|
||||
INSTANTIATE(2, NPB, N, K) \
|
||||
INSTANTIATE(3, NPB, N, K) \
|
||||
INSTANTIATE(4, NPB, N, K) \
|
||||
INSTANTIATE(5, NPB, N, K) \
|
||||
INSTANTIATE(6, NPB, N, K) \
|
||||
INSTANTIATE(7, NPB, N, K) \
|
||||
INSTANTIATE(8, NPB, N, K) \
|
||||
INSTANTIATE(9, NPB, N, K) \
|
||||
INSTANTIATE(10, NPB, N, K) \
|
||||
INSTANTIATE(11, NPB, N, K) \
|
||||
INSTANTIATE(12, NPB, N, K) \
|
||||
INSTANTIATE(13, NPB, N, K) \
|
||||
INSTANTIATE(14, NPB, N, K) \
|
||||
INSTANTIATE(15, NPB, N, K) \
|
||||
INSTANTIATE(16, NPB, N, K) \
|
||||
INSTANTIATE(17, NPB, N, K) \
|
||||
INSTANTIATE(18, NPB, N, K) \
|
||||
INSTANTIATE(19, NPB, N, K) \
|
||||
INSTANTIATE(20, NPB, N, K) \
|
||||
INSTANTIATE(21, NPB, N, K) \
|
||||
INSTANTIATE(22, NPB, N, K) \
|
||||
INSTANTIATE(23, NPB, N, K) \
|
||||
INSTANTIATE(24, NPB, N, K) \
|
||||
INSTANTIATE(25, NPB, N, K) \
|
||||
INSTANTIATE(26, NPB, N, K) \
|
||||
INSTANTIATE(27, NPB, N, K) \
|
||||
INSTANTIATE(28, NPB, N, K) \
|
||||
INSTANTIATE(29, NPB, N, K) \
|
||||
INSTANTIATE(30, NPB, N, K) \
|
||||
INSTANTIATE(31, NPB, N, K) \
|
||||
INSTANTIATE(32, NPB, N, K)
|
||||
|
||||
INSTANTIATE_ALL_M(4, 768, 12288)
|
||||
INSTANTIATE_ALL_M(4, 1536, 12288)
|
||||
INSTANTIATE_ALL_M(2, 6144, 12288)
|
||||
// LL-mode (M<=8 wiring guard) backbone shapes, B300 sweep vs cuBLAS:
|
||||
// q_b_proj (2048, 2048): 1.67x/1.29x/1.15x at M=4/6/8 (NPB=4 within
|
||||
// 0.1us of per-M best)
|
||||
// shared-expert gate_up (512, 6144): 1.95x/1.58x/1.40x at M=4/6/8
|
||||
// cuBLAS keeps qkv_a (2624,6144) and o_proj (6144,2048) — already at
|
||||
// 3.4-3.8 TB/s there; the GEMV loses on activation re-reads.
|
||||
INSTANTIATE_ALL_M(4, 2048, 2048)
|
||||
INSTANTIATE_ALL_M(4, 512, 6144)
|
||||
// fused_qkv_a (2624, 6144), 32MB: skinny wins ONLY at M<=2 (B300: M=1
|
||||
// 6.99us vs cuBLAS 9.21 = 1.32x, M=2 1.24x; M>=4 cuBLAS holds at 3.5TB/s
|
||||
// and every alternative loses — cublasLt top-8 3.6TB/s wall, DeepGEMM
|
||||
// 0.71x, wmma+cp.async custom 0.30x pending a TMA rewrite).
|
||||
INSTANTIATE_ALL_M(4, 2624, 6144)
|
||||
// DSv3.2 (TP8) siblings of the GLM shapes above, same dual-chip matrix:
|
||||
// fused_qkv_a (2112, 7168), 30MB: wins M<=2 (M=1 1.30-1.34x)
|
||||
// MTP eh_proj (7168, 14336), 205MB: wins M<=2 (M=1 1.12-1.16x)
|
||||
INSTANTIATE_ALL_M(4, 2112, 7168)
|
||||
INSTANTIATE_ALL_M(2, 7168, 14336)
|
||||
|
||||
#undef INSTANTIATE_ALL_M
|
||||
#undef INSTANTIATE
|
||||
@@ -0,0 +1,170 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
#include <torch/csrc/stable/library.h>
|
||||
#include <torch/csrc/stable/tensor.h>
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
|
||||
#include "core/registration.h"
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <stdexcept>
|
||||
|
||||
namespace {
|
||||
|
||||
inline int getSMVersion() {
|
||||
auto* props = get_device_prop();
|
||||
return props->major * 10 + props->minor;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
static constexpr int SKINNY_MAX_TOKENS = 32;
|
||||
|
||||
// Supported (N, K) pairs (must match the instantiations in
|
||||
// bf16_skinny_gemm.cu): eh_proj shard TP8 / TP4 / unsharded.
|
||||
static inline bool bf16_skinny_gemm_supported(int n, int k) {
|
||||
if (k == 12288 && (n == 768 || n == 1536 || n == 6144)) return true;
|
||||
// LL-mode backbone shapes (wire callers with an M <= 8 guard; the GEMV
|
||||
// family loses to cuBLAS at larger M).
|
||||
if (k == 2048 && n == 2048) return true; // q_b_proj (TP8)
|
||||
if (k == 6144 && n == 2624) return true; // fused_qkv_a (wire M <= 2 only)
|
||||
if (k == 7168 && n == 2112) return true; // DSv3.2 fused_qkv_a (M <= 2)
|
||||
if (k == 14336 && n == 7168) return true; // DSv3.2 eh_proj (M <= 2)
|
||||
if (k == 6144 && n == 512) return true; // shared-expert gate_up (TP8)
|
||||
return false;
|
||||
}
|
||||
|
||||
// Forward declarations - template params must match bf16_skinny_gemm.cu
|
||||
template <int kBlockSize, int kNPB, int kNumTokens, int kN, int kK>
|
||||
void invokeBf16SkinnyGemm(__nv_bfloat16* output, __nv_bfloat16 const* mat_a,
|
||||
__nv_bfloat16 const* mat_b, int64_t out_stride,
|
||||
cudaStream_t stream);
|
||||
|
||||
template <int kNPB, int kN, int kK, int kBegin, int kEnd>
|
||||
struct SkinnyLoopUnroller {
|
||||
static void unroll(int num_tokens, __nv_bfloat16* output,
|
||||
__nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b,
|
||||
int64_t out_stride, cudaStream_t stream) {
|
||||
if (num_tokens == kBegin) {
|
||||
invokeBf16SkinnyGemm<128, kNPB, kBegin, kN, kK>(output, mat_a, mat_b,
|
||||
out_stride, stream);
|
||||
} else {
|
||||
SkinnyLoopUnroller<kNPB, kN, kK, kBegin + 1, kEnd>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <int kNPB, int kN, int kK, int kEnd>
|
||||
struct SkinnyLoopUnroller<kNPB, kN, kK, kEnd, kEnd> {
|
||||
static void unroll(int num_tokens, __nv_bfloat16* output,
|
||||
__nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b,
|
||||
int64_t out_stride, cudaStream_t stream) {
|
||||
if (num_tokens == kEnd) {
|
||||
invokeBf16SkinnyGemm<128, kNPB, kEnd, kN, kK>(output, mat_a, mat_b,
|
||||
out_stride, stream);
|
||||
} else {
|
||||
throw std::invalid_argument(
|
||||
"bf16_skinny_gemm: num_tokens must be in [1, 32]");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
static void dispatchBf16SkinnyGemm(int n, int k, int num_tokens,
|
||||
__nv_bfloat16* output,
|
||||
__nv_bfloat16 const* mat_a,
|
||||
__nv_bfloat16 const* mat_b,
|
||||
int64_t out_stride, cudaStream_t stream) {
|
||||
if (n == 768 && k == 12288) {
|
||||
SkinnyLoopUnroller<4, 768, 12288, 1, SKINNY_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
} else if (n == 1536 && k == 12288) {
|
||||
SkinnyLoopUnroller<4, 1536, 12288, 1, SKINNY_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
} else if (n == 6144 && k == 12288) {
|
||||
SkinnyLoopUnroller<2, 6144, 12288, 1, SKINNY_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
} else if (n == 2048 && k == 2048) {
|
||||
SkinnyLoopUnroller<4, 2048, 2048, 1, SKINNY_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
} else if (n == 2624 && k == 6144) {
|
||||
SkinnyLoopUnroller<4, 2624, 6144, 1, SKINNY_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
} else if (n == 2112 && k == 7168) {
|
||||
SkinnyLoopUnroller<4, 2112, 7168, 1, SKINNY_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
} else if (n == 7168 && k == 14336) {
|
||||
SkinnyLoopUnroller<2, 7168, 14336, 1, SKINNY_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
} else if (n == 512 && k == 6144) {
|
||||
SkinnyLoopUnroller<4, 512, 6144, 1, SKINNY_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
} else {
|
||||
throw std::invalid_argument("bf16_skinny_gemm: unsupported (N, K) pair");
|
||||
}
|
||||
}
|
||||
|
||||
void bf16_skinny_gemm(
|
||||
torch::stable::Tensor& output, // [num_tokens, N] bf16
|
||||
torch::stable::Tensor const& mat_a, // [num_tokens, K] bf16
|
||||
torch::stable::Tensor const& mat_b // [N, K] bf16
|
||||
) {
|
||||
STD_TORCH_CHECK(output.dim() == 2 && mat_a.dim() == 2 && mat_b.dim() == 2);
|
||||
STD_TORCH_CHECK(output.is_cuda() && mat_a.is_cuda() && mat_b.is_cuda(),
|
||||
"bf16_skinny_gemm: all tensors must be CUDA tensors");
|
||||
STD_TORCH_CHECK(output.get_device_index() == mat_a.get_device_index() &&
|
||||
output.get_device_index() == mat_b.get_device_index(),
|
||||
"bf16_skinny_gemm: all tensors must be on the same device");
|
||||
STD_TORCH_CHECK(mat_a.is_contiguous() && mat_b.is_contiguous(),
|
||||
"bf16_skinny_gemm: inputs must be contiguous");
|
||||
// Output may be a column-slice view of a wider padded buffer: unit column
|
||||
// stride, row stride >= N (rows must not overlap).
|
||||
STD_TORCH_CHECK(output.stride(1) == 1,
|
||||
"bf16_skinny_gemm: output columns must be contiguous");
|
||||
|
||||
const int num_tokens = mat_a.size(0);
|
||||
const int n = mat_b.size(0);
|
||||
const int k = mat_a.size(1);
|
||||
|
||||
STD_TORCH_CHECK(output.size(0) == num_tokens && output.size(1) == n,
|
||||
"bf16_skinny_gemm: output must be [num_tokens, N]");
|
||||
STD_TORCH_CHECK(mat_b.size(1) == k,
|
||||
"bf16_skinny_gemm: mat_a and mat_b must share K");
|
||||
STD_TORCH_CHECK(bf16_skinny_gemm_supported(n, k),
|
||||
"bf16_skinny_gemm: unsupported (N, K) pair");
|
||||
const int64_t out_stride = output.stride(0);
|
||||
STD_TORCH_CHECK(num_tokens <= 1 || out_stride >= n,
|
||||
"bf16_skinny_gemm: output rows overlap");
|
||||
STD_TORCH_CHECK(num_tokens >= 0 && num_tokens <= SKINNY_MAX_TOKENS,
|
||||
"bf16_skinny_gemm: num_tokens must be in [0, 32]");
|
||||
STD_TORCH_CHECK(
|
||||
mat_a.scalar_type() == torch::headeronly::ScalarType::BFloat16 &&
|
||||
mat_b.scalar_type() == torch::headeronly::ScalarType::BFloat16 &&
|
||||
output.scalar_type() == torch::headeronly::ScalarType::BFloat16,
|
||||
"bf16_skinny_gemm: all tensors must be bfloat16");
|
||||
|
||||
// Empty batch (e.g. an empty rank at a DP/PP boundary): nothing to compute.
|
||||
if (num_tokens == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
mat_a.get_device_index());
|
||||
STD_TORCH_CHECK(getSMVersion() >= 90, "bf16_skinny_gemm: requires SM90+");
|
||||
|
||||
auto stream = get_current_cuda_stream(mat_a.get_device_index());
|
||||
dispatchBf16SkinnyGemm(
|
||||
n, k, num_tokens,
|
||||
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), out_stride,
|
||||
stream);
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) {
|
||||
m.impl("bf16_skinny_gemm", TORCH_BOX(&bf16_skinny_gemm));
|
||||
}
|
||||
@@ -391,7 +391,7 @@ struct MmaComputer {
|
||||
static constexpr int n_iter_cnt =
|
||||
(tile_n + 7) /
|
||||
8; // Possible to have non-1 n_iter_cnt for ab_swap m16 case.
|
||||
static_assert(m_iter_cnt == 1);
|
||||
static_assert(m_iter_cnt == 1 || m_iter_cnt == 2);
|
||||
static_assert(n_iter_cnt == 1 || n_iter_cnt == 2);
|
||||
|
||||
__device__ MmaComputer(bf16_t* gmem_c_local_, bf16_t* smem_a_,
|
||||
@@ -416,13 +416,18 @@ struct MmaComputer {
|
||||
public:
|
||||
__device__ void prepare() {
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
|
||||
// Fragment addressing is per 16-row ldmatrix tile; m_iter selects the
|
||||
// 16-row half within tile_m.
|
||||
#pragma unroll
|
||||
for (int i = 0; i < k_phase_cnt; i++) {
|
||||
int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256;
|
||||
int m_idx = linear_idx % tile_m;
|
||||
int k_idx = linear_idx / tile_m + warp_k_offset_in_tile_k;
|
||||
k_idx = apply_swizzle_343_on_elem_row_col<bf16_t>(m_idx, k_idx);
|
||||
a_smem_offsets[0][i] = m_idx * tile_k + k_idx;
|
||||
for (int m = 0; m < m_iter_cnt; m++) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < k_phase_cnt; i++) {
|
||||
int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256;
|
||||
int m_idx = linear_idx % 16 + m * 16;
|
||||
int k_idx = linear_idx / 16 + warp_k_offset_in_tile_k;
|
||||
k_idx = apply_swizzle_343_on_elem_row_col<bf16_t>(m_idx, k_idx);
|
||||
a_smem_offsets[m][i] = m_idx * tile_k + k_idx;
|
||||
}
|
||||
}
|
||||
#pragma unroll
|
||||
for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) {
|
||||
@@ -446,11 +451,14 @@ struct MmaComputer {
|
||||
wait_barrier(smem_barrier + 0 + stage_idx * 2, phase_bit);
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < k_phase_cnt; i++) {
|
||||
int smem_offset = a_smem_offsets[0][i];
|
||||
bf16_t* smem_ptr_this_iter =
|
||||
smem_a + stage_idx * tile_m * tile_k + smem_offset;
|
||||
ldsm_x4(smem_ptr_this_iter, reinterpret_cast<uint32_t*>(a_reg[0][i]));
|
||||
for (int m = 0; m < m_iter_cnt; m++) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < k_phase_cnt; i++) {
|
||||
int smem_offset = a_smem_offsets[m][i];
|
||||
bf16_t* smem_ptr_this_iter =
|
||||
smem_a + stage_idx * tile_m * tile_k + smem_offset;
|
||||
ldsm_x4(smem_ptr_this_iter, reinterpret_cast<uint32_t*>(a_reg[m][i]));
|
||||
}
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
@@ -469,9 +477,12 @@ struct MmaComputer {
|
||||
for (int k_iter_idx = 0; k_iter_idx < k_phase_cnt; k_iter_idx++) {
|
||||
#pragma unroll
|
||||
for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) {
|
||||
hmma_16_8_16_f32acc_bf16ab(
|
||||
acc_reg[0][n_iter_idx], a_reg[0][k_iter_idx],
|
||||
b_reg[n_iter_idx][k_iter_idx], acc_reg[0][n_iter_idx]);
|
||||
#pragma unroll
|
||||
for (int m = 0; m < m_iter_cnt; m++) {
|
||||
hmma_16_8_16_f32acc_bf16ab(
|
||||
acc_reg[m][n_iter_idx], a_reg[m][k_iter_idx],
|
||||
b_reg[n_iter_idx][k_iter_idx], acc_reg[m][n_iter_idx]);
|
||||
}
|
||||
}
|
||||
}
|
||||
::arrive_barrier(smem_barrier + 1 + stage_idx * 2);
|
||||
@@ -486,14 +497,14 @@ struct MmaComputer {
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
|
||||
asm volatile("bar.sync %0, %1;" : : "r"(1), "r"(thread_cnt));
|
||||
// reorganize the acc_reg
|
||||
constexpr int thread_m = 2;
|
||||
constexpr int thread_m = 2 * m_iter_cnt;
|
||||
constexpr int thread_n = 2 * n_iter_cnt;
|
||||
constexpr int cta_mma_n = n_iter_cnt * 8;
|
||||
float acc_reg_reorg[thread_m][thread_n];
|
||||
|
||||
for (int i = 0; i < thread_m; i++) {
|
||||
for (int j = 0; j < thread_n; j++) {
|
||||
acc_reg_reorg[i][j] = acc_reg[0][j / 2][(j % 2) + (i * 2)];
|
||||
acc_reg_reorg[i][j] = acc_reg[i / 2][j / 2][(j % 2) + (i % 2) * 2];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -514,7 +525,8 @@ struct MmaComputer {
|
||||
for (int m_idx_thread = 0; m_idx_thread < thread_m; m_idx_thread++) {
|
||||
#pragma unroll
|
||||
for (int n_idx_thread = 0; n_idx_thread < thread_n; n_idx_thread++) {
|
||||
int m_idx = (lane_idx / 4) + m_idx_thread * 8;
|
||||
int m_idx =
|
||||
(lane_idx / 4) + (m_idx_thread % 2) * 8 + (m_idx_thread / 2) * 16;
|
||||
int n_idx =
|
||||
((lane_idx % 4) * 2) + (n_idx_thread % 2) + (n_idx_thread / 2) * 8;
|
||||
smem_c[cosize_smem_c * warp_idx + smem_c_index_func(m_idx, n_idx)] =
|
||||
@@ -587,7 +599,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
|
||||
static_assert(
|
||||
tile_k == 128 || tile_k == 256 || tile_k == 512 ||
|
||||
tile_k == 1024); // tile_k must be larger than 64 since 4 warp splitK.
|
||||
static_assert(tile_m == 16);
|
||||
static_assert(tile_m == 16 || tile_m == 32);
|
||||
constexpr int g2s_vec_bytes = 16;
|
||||
constexpr int a_elem_bytes = 2;
|
||||
constexpr int b_elem_bytes = 2;
|
||||
@@ -647,7 +659,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename T, int kHdIn, int kHdOut, int kTileN>
|
||||
template <typename T, int kHdIn, int kHdOut, int kTileN, int kTileM = 16>
|
||||
void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
|
||||
cudaStream_t const stream) {
|
||||
constexpr int gemm_m = kHdOut; // 2112
|
||||
@@ -655,7 +667,7 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
|
||||
constexpr int gemm_k = kHdIn; // 7168
|
||||
constexpr int batch_size = 1;
|
||||
std::swap(mat_a, mat_b);
|
||||
constexpr int tile_m = 16;
|
||||
constexpr int tile_m = kTileM;
|
||||
constexpr int tile_n = kTileN; // 8 or 16
|
||||
constexpr int tile_k = std::max(256, 1024 / tile_n); // 256
|
||||
constexpr int max_stage_cnt =
|
||||
@@ -702,6 +714,44 @@ template void invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 16>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
|
||||
cudaStream_t);
|
||||
|
||||
// GLM-5.2 fused_qkv_a (K=6144 -> N=2624). tile_m=32 so the grid is
|
||||
// 2624/32 = 82 CTAs, a single wave on B300 (148 SMs); tile_m=16's 164 CTAs
|
||||
// straddle two waves and drop to 2.9 TB/s vs 3.9-4.0 here. Beats cuBLAS
|
||||
// at every decode M: 1.11-1.15x for M<=8 (tile_n=8), 1.12x at M=16
|
||||
// (tile_n=16). The M<=2 dispatch still belongs to bf16_skinny_gemm
|
||||
// (4.5 TB/s).
|
||||
template void invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 8, 32>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
|
||||
cudaStream_t);
|
||||
|
||||
template void invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 16, 32>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
|
||||
cudaStream_t);
|
||||
|
||||
// GLM-5.2 q_b_proj TP8 (K=2048 -> N=2048). tile_m=16 keeps 128 CTAs (already
|
||||
// a single wave) and measures ahead of tile_m=32 here. Beats cuBLAS
|
||||
// 1.79-1.87x at M=3..8 (tile_n=8) and 1.19-1.30x at M=9..16 (tile_n=16);
|
||||
// M<=2 belongs to bf16_skinny_gemm, M>=20 to cuBLAS.
|
||||
template void invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 8>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
|
||||
cudaStream_t);
|
||||
|
||||
template void invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 16>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
|
||||
cudaStream_t);
|
||||
|
||||
// DSv3.2 q_b_proj TP8 (K=1536 -> N=3072). tile_m=32 keeps 96 CTAs (single
|
||||
// wave; tile_m=16's 192 straddle two). Beats cuBLAS 1.6-1.8x at M=1..8 and
|
||||
// 1.05-1.18x at M=9..16 on B300/B200; the whole 1..16 range dispatches here
|
||||
// (no skinny tier: K=1536 does not fit the GEMV's 128x8 K-step).
|
||||
template void invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 8, 32>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
|
||||
cudaStream_t);
|
||||
|
||||
template void invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 16, 32>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
|
||||
cudaStream_t);
|
||||
|
||||
void dsv3_fused_a_gemm(torch::stable::Tensor& output,
|
||||
torch::stable::Tensor const& mat_a,
|
||||
torch::stable::Tensor const& mat_b) {
|
||||
@@ -710,12 +760,15 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output,
|
||||
int const hd_in = mat_a.size(1);
|
||||
int const hd_out = mat_b.size(1);
|
||||
|
||||
constexpr int kHdIn = 7168;
|
||||
constexpr int kHdOut = 2112;
|
||||
bool const is_dsv3 = hd_in == 7168 && hd_out == 2112;
|
||||
bool const is_glm = hd_in == 6144 && hd_out == 2624;
|
||||
bool const is_glm_qb = hd_in == 2048 && hd_out == 2048;
|
||||
bool const is_ds_qb = hd_in == 1536 && hd_out == 3072;
|
||||
STD_TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16,
|
||||
"required 1 <= mat_a.shape[0] <= 16");
|
||||
STD_TORCH_CHECK(hd_in == kHdIn, "required mat_a.shape[1] == 7168");
|
||||
STD_TORCH_CHECK(hd_out == kHdOut, "required mat_b.shape[1] == 2112");
|
||||
STD_TORCH_CHECK(is_dsv3 || is_glm || is_glm_qb || is_ds_qb,
|
||||
"supported (hd_in, hd_out): (7168, 2112), (6144, 2624), "
|
||||
"(2048, 2048), (1536, 3072)");
|
||||
STD_TORCH_CHECK(output.size(0) == num_tokens,
|
||||
"required output.shape[0] == mat_a.shape[0]");
|
||||
STD_TORCH_CHECK(output.size(1) == hd_out,
|
||||
@@ -738,18 +791,41 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output,
|
||||
STD_TORCH_CHECK(getSMVersion() >= 90, "required CUDA ARCH >= SM_90");
|
||||
|
||||
auto stream = get_current_cuda_stream(mat_a.get_device_index());
|
||||
if (num_tokens <= 8) {
|
||||
invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 8>(
|
||||
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), num_tokens,
|
||||
stream);
|
||||
auto* out = reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr());
|
||||
auto* a = reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr());
|
||||
auto* b = reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr());
|
||||
if (is_dsv3) {
|
||||
if (num_tokens <= 8) {
|
||||
invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 8>(out, a, b, num_tokens,
|
||||
stream);
|
||||
} else {
|
||||
invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 16>(out, a, b, num_tokens,
|
||||
stream);
|
||||
}
|
||||
} else if (is_glm) {
|
||||
if (num_tokens <= 8) {
|
||||
invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 8, 32>(out, a, b, num_tokens,
|
||||
stream);
|
||||
} else {
|
||||
invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 16, 32>(out, a, b, num_tokens,
|
||||
stream);
|
||||
}
|
||||
} else if (is_glm_qb) {
|
||||
if (num_tokens <= 8) {
|
||||
invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 8>(out, a, b, num_tokens,
|
||||
stream);
|
||||
} else {
|
||||
invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 16>(out, a, b, num_tokens,
|
||||
stream);
|
||||
}
|
||||
} else {
|
||||
invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 16>(
|
||||
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), num_tokens,
|
||||
stream);
|
||||
if (num_tokens <= 8) {
|
||||
invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 8, 32>(out, a, b, num_tokens,
|
||||
stream);
|
||||
} else {
|
||||
invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 16, 32>(out, a, b, num_tokens,
|
||||
stream);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -49,6 +49,11 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
||||
static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD,
|
||||
"Vec size is not matched.");
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
cudaGridDependencySynchronize();
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
|
||||
// Precompute SF layout parameter (constant for entire kernel).
|
||||
int32_t const numKTiles = (outputCols + 63) / 64;
|
||||
|
||||
@@ -123,6 +128,11 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
||||
static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD,
|
||||
"Vec size is not matched.");
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
cudaGridDependencySynchronize();
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
|
||||
int32_t const colIdx = blockDim.x * blockIdx.y + threadIdx.x;
|
||||
int elem_idx = colIdx * CVT_FP4_ELTS_PER_THREAD;
|
||||
|
||||
@@ -201,6 +211,8 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
auto stream = get_current_cuda_stream(input.get_device_index());
|
||||
auto* device_props = get_device_prop();
|
||||
int const sm_version = device_props->major * 10 + device_props->minor;
|
||||
|
||||
int output_sf_n_unpadded = int(output_n / CVT_FP4_SF_VEC_SIZE);
|
||||
|
||||
@@ -224,10 +236,21 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
|
||||
input.scalar_type(), "nvfp4_quant_kernel", [&] {
|
||||
using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type;
|
||||
auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
|
||||
vllm::cvt_fp16_to_fp4<cuda_type, false><<<grid, block, 0, stream>>>(
|
||||
m, n, output_n, num_padded_cols, input_ptr, input_sf_ptr,
|
||||
reinterpret_cast<uint32_t*>(output_ptr),
|
||||
reinterpret_cast<uint32_t*>(sf_out));
|
||||
cudaLaunchConfig_t config = {};
|
||||
config.gridDim = grid;
|
||||
config.blockDim = block;
|
||||
config.dynamicSmemBytes = 0;
|
||||
config.stream = stream;
|
||||
cudaLaunchAttribute attrs[1];
|
||||
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
|
||||
attrs[0].val.programmaticStreamSerializationAllowed = 1;
|
||||
config.numAttrs = (sm_version >= 90) ? 1 : 0;
|
||||
config.attrs = attrs;
|
||||
cudaLaunchKernelEx(&config, vllm::cvt_fp16_to_fp4<cuda_type, false>,
|
||||
m, n, output_n, num_padded_cols, input_ptr,
|
||||
input_sf_ptr,
|
||||
reinterpret_cast<uint32_t*>(output_ptr),
|
||||
reinterpret_cast<uint32_t*>(sf_out));
|
||||
});
|
||||
} else {
|
||||
int num_packed_cols = output_n / CVT_FP4_ELTS_PER_THREAD;
|
||||
@@ -240,12 +263,21 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
|
||||
input.scalar_type(), "nvfp4_quant_kernel", [&] {
|
||||
using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type;
|
||||
auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
|
||||
vllm::cvt_fp16_to_fp4_sf_major<cuda_type, false>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
m, n, output_n, output_sf_n_unpadded, num_packed_cols,
|
||||
input_ptr, input_sf_ptr,
|
||||
reinterpret_cast<uint32_t*>(output_ptr),
|
||||
reinterpret_cast<uint32_t*>(sf_out));
|
||||
cudaLaunchConfig_t config = {};
|
||||
config.gridDim = grid;
|
||||
config.blockDim = block;
|
||||
config.dynamicSmemBytes = 0;
|
||||
config.stream = stream;
|
||||
cudaLaunchAttribute attrs[1];
|
||||
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
|
||||
attrs[0].val.programmaticStreamSerializationAllowed = 1;
|
||||
config.numAttrs = (sm_version >= 90) ? 1 : 0;
|
||||
config.attrs = attrs;
|
||||
cudaLaunchKernelEx(
|
||||
&config, vllm::cvt_fp16_to_fp4_sf_major<cuda_type, false>, m, n,
|
||||
output_n, output_sf_n_unpadded, num_packed_cols, input_ptr,
|
||||
input_sf_ptr, reinterpret_cast<uint32_t*>(output_ptr),
|
||||
reinterpret_cast<uint32_t*>(sf_out));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -330,6 +330,10 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
// conditionally compiled so impl registration is in source file
|
||||
ops.def("fp32_router_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()");
|
||||
|
||||
// BF16 skinny GEMM (M<=32, weight-BW-bound shapes, e.g. MTP eh_proj).
|
||||
// conditionally compiled so impl registration is in source file
|
||||
ops.def("bf16_skinny_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()");
|
||||
|
||||
// reorder weight for AllSpark Ampere W8A16 Fused Gemm kernel
|
||||
ops.def(
|
||||
"rearrange_kn_weight_as_n32k16_order(Tensor b_qweight, Tensor b_scales, "
|
||||
|
||||
Reference in New Issue
Block a user