[K3 Perf] Enable DSV3 GEMM for inner-contiguous and row-strided tensors, 12%~81% kernel performance improvement (#54565)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-09-02 12:22:40 -07:00
committed by GitHub
parent 0e3ac4907d
commit a56654d6de
3 changed files with 47 additions and 34 deletions
+34 -26
View File
@@ -326,7 +326,7 @@ struct GmemLoaderB {
#endif
}
__device__ void issue_mainloop() {
__device__ void issue_mainloop(int row_stride) {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
cudaGridDependencySynchronize();
#pragma unroll 1
@@ -350,7 +350,7 @@ struct GmemLoaderB {
int linear_idx = local_tid * vec_elems + i * thread_cnt * vec_elems;
int n_idx = linear_idx / tile_k;
int k_idx = linear_idx % tile_k;
int gmem_offset = n_idx * gemm_k + k_project(k_idx);
int gmem_offset = n_idx * row_stride + k_project(k_idx);
bf16_t const* gmem_ptr_this_iter = gmem_b + gmem_offset;
ldgsts_128(gmem_ptr_this_iter, smem_ptr_this_iter, preds[i]);
}
@@ -585,7 +585,8 @@ struct MmaComputer {
template <int batch_size, int gemm_m, int gemm_k, int tile_m, int tile_n,
int tile_k, int stage_cnt>
__global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
bf16_t* output, bf16_t const* mat_a, bf16_t const* mat_b, int gemm_n) {
bf16_t* output, bf16_t const* mat_a, bf16_t const* mat_b, int gemm_n,
int input_row_stride) {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
constexpr int load_thread_cnt = 128;
constexpr int compute_thread_cnt = 128;
@@ -621,7 +622,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
int cta_m_idx = tile_m * blockIdx.x;
int cta_n_idx = tile_n * blockIdx.y;
bf16_t const* gmem_a_local = mat_a + cta_m_idx * gemm_k;
bf16_t const* gmem_b_local = mat_b + cta_n_idx * gemm_k;
bf16_t const* gmem_b_local = mat_b + cta_n_idx * input_row_stride;
bf16_t* gmem_c_local = output + cta_n_idx * gemm_m + cta_m_idx;
int warp_idx = __shfl_sync(0xffffffff, threadIdx.x / 32, 0);
@@ -645,7 +646,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
GmemLoaderB<gemm_k, tile_n, tile_k, stage_cnt> b_loader(
gmem_b_local, smem_b, smem_barrier, gemm_n);
b_loader.prepare();
b_loader.issue_mainloop();
b_loader.issue_mainloop(input_row_stride);
} else {
MmaComputer<gemm_m, gemm_k, tile_m, tile_n, tile_k, stage_cnt> mma_computer(
gmem_c_local, smem_a, smem_b, smem_barrier, warp_idx, gemm_n);
@@ -660,7 +661,8 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
template <typename T, int kHdIn, int kHdOut, int kTileN, int kTileK = 256,
int kTileM = 16>
void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
cudaStream_t const stream, bool enable_pdl) {
int input_row_stride, cudaStream_t const stream,
bool enable_pdl) {
constexpr int gemm_m = kHdOut;
int const gemm_n = num_tokens;
constexpr int gemm_k = kHdIn;
@@ -703,19 +705,19 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
cudaLaunchKernelEx(&config,
fused_a_gemm_kernel<batch_size, gemm_m, gemm_k, tile_m,
tile_n, tile_k, stage_cnt>,
output, mat_a, mat_b, gemm_n);
output, mat_a, mat_b, gemm_n, input_row_stride);
}
template <typename T, int kHdIn, int kHdOut, int kTileK = 256, int kTileM = 16>
void invokeFusedAGemmForTokens(T* output, T const* mat_a, T const* mat_b,
int num_tokens, cudaStream_t const stream,
bool enable_pdl) {
int num_tokens, int input_row_stride,
cudaStream_t const stream, bool enable_pdl) {
if (num_tokens <= 8) {
invokeFusedAGemm<T, kHdIn, kHdOut, 8, kTileK, kTileM>(
output, mat_a, mat_b, num_tokens, stream, enable_pdl);
output, mat_a, mat_b, num_tokens, input_row_stride, stream, enable_pdl);
} else {
invokeFusedAGemm<T, kHdIn, kHdOut, 16, kTileK, kTileM>(
output, mat_a, mat_b, num_tokens, stream, enable_pdl);
output, mat_a, mat_b, num_tokens, input_row_stride, stream, enable_pdl);
}
}
@@ -740,11 +742,9 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output,
mat_a.get_device_index() == output.get_device_index(),
"mat_a, mat_b, and output must be on the same device");
// The kernels index global memory with raw pointers and packed strides, so
// reject any padded or transposed view rather than reading out of bounds.
STD_TORCH_CHECK(
mat_a.stride(0) == hd_in && mat_a.stride(1) == 1,
"mat_a must be a packed row-major [num_tokens, hd_in] tensor");
mat_a.stride(1) == 1 && (num_tokens == 1 || mat_a.stride(0) % 8 == 0),
"mat_a must have unit inner stride and 16-byte aligned rows");
STD_TORCH_CHECK(
output.stride(0) == hd_out && output.stride(1) == 1,
"output must be a packed row-major [num_tokens, hd_out] tensor");
@@ -770,12 +770,14 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output,
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr());
auto const* mat_b_ptr =
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr());
int const input_row_stride = mat_a.stride(0);
#define DISPATCH_DSV3_SHAPE(HD_IN, HD_OUT) \
if (hd_in == HD_IN && hd_out == HD_OUT) { \
invokeFusedAGemmForTokens<__nv_bfloat16, HD_IN, HD_OUT>( \
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl); \
return; \
#define DISPATCH_DSV3_SHAPE(HD_IN, HD_OUT) \
if (hd_in == HD_IN && hd_out == HD_OUT) { \
invokeFusedAGemmForTokens<__nv_bfloat16, HD_IN, HD_OUT>( \
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, input_row_stride, \
stream, enable_pdl); \
return; \
}
// Shapes the Kimi-K3 selector routes to dsv3_fused_a (see the dsv3 winners
@@ -795,7 +797,8 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output,
if (hd_in == 6144 && hd_out == 2624) {
invokeFusedAGemmForTokens<__nv_bfloat16, 6144, 2624, 256, 32>(
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl);
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, input_row_stride, stream,
enable_pdl);
return;
}
DISPATCH_DSV3_SHAPE(2048, 2048)
@@ -821,30 +824,35 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output,
if (hd_in == 128 && hd_out == 1536) {
invokeFusedAGemmForTokens<__nv_bfloat16, 128, 1536, 128>(
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl);
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, input_row_stride, stream,
enable_pdl);
return;
}
if (hd_in == 128 && hd_out == 3072) {
invokeFusedAGemmForTokens<__nv_bfloat16, 128, 3072, 128>(
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl);
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, input_row_stride, stream,
enable_pdl);
return;
}
// TP16 KDA f_b_proj and shared_expert down_proj. Neither hd_in is a multiple
// of 256, so both need the 128 tile_k.
if (hd_in == 128 && hd_out == 768) {
invokeFusedAGemmForTokens<__nv_bfloat16, 128, 768, 128>(
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl);
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, input_row_stride, stream,
enable_pdl);
return;
}
if (hd_in == 384 && hd_out == 7168) {
invokeFusedAGemmForTokens<__nv_bfloat16, 384, 7168, 128>(
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl);
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, input_row_stride, stream,
enable_pdl);
return;
}
#ifdef VLLM_K3_BENCH_SHAPES
if (hd_in == 4224 && hd_out == 7168) {
invokeFusedAGemmForTokens<__nv_bfloat16, 4224, 7168, 128>(
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl);
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, input_row_stride, stream,
enable_pdl);
return;
}
#endif
+6 -6
View File
@@ -996,20 +996,20 @@ def test_glm_dsv3_selected_shapes(
assert cosine > 0.999
def test_nonpacked_single_token_dsv3_falls_back() -> None:
@pytest.mark.parametrize("num_tokens", [1, 4])
def test_kda_f_b_nonpacked_dsv3_dispatches(num_tokens: int) -> None:
_require_sm103_and_dsv3()
n, k = 1536, 128
storage = torch.randn(1, k + 16, dtype=torch.bfloat16, device="cuda")
x = storage[:, :k]
storage = torch.randn(num_tokens, 6288, dtype=torch.bfloat16, device="cuda")
x = storage[:, 6144 : 6144 + k]
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
spec = k3_gemm.KIMI_K3_PROJECTIONS[(n, k)]
method = k3_gemm.KimiK3LowLatencyLinearMethod(
k3_gemm._build_plan(spec), k3_gemm._build_residual_plan(spec)
)
assert x.is_contiguous()
assert x.stride() == (k + 16, 1)
assert not k3_gemm._runtime_ok(x, weight) # strict guard rejects the view
assert x.stride() == (6288, 1)
assert k3_gemm._runtime_ok(x, weight)
output = method.apply(SimpleNamespace(weight=weight), x)
reference = torch.nn.functional.linear(x, weight)
@@ -663,7 +663,9 @@ def _is_packed_row_major(tensor: torch.Tensor) -> bool:
def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool:
return (
_is_packed_row_major(x)
x.dim() == 2
and x.stride(1) == 1
and (x.shape[0] == 1 or x.stride(0) % 8 == 0)
and _is_packed_row_major(weight)
and x.dtype == torch.bfloat16
and weight.dtype == torch.bfloat16
@@ -676,7 +678,8 @@ def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool:
def _residual_ok(x: torch.Tensor, weight: torch.Tensor, residual: torch.Tensor) -> bool:
return (
residual.dim() == 2
_is_packed_row_major(x)
and residual.dim() == 2
and residual.dtype == torch.bfloat16
and residual.is_cuda
and residual.device == x.device
@@ -693,6 +696,8 @@ def _run_plan(
return None
backend, config = entry
if backend == "cute":
if x.shape[0] != 1 and not _is_packed_row_major(x):
return None
if not shape_dynamic_skinny_gemm.is_available():
return None
return shape_dynamic_skinny_gemm(x, weight, config, None)