[Kernel] Extend CuTe DSL skinny GEMM to GLM-5.2 (#49791)

Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
xiaozhoupy
2026-08-03 16:49:41 -07:00
committed by GitHub
co-authored by OpenAI Codex Jee Jee Li
parent 6a9109d865
commit c810937573
7 changed files with 619 additions and 51 deletions
+42 -24
View File
@@ -391,7 +391,7 @@ struct MmaComputer {
static constexpr int n_iter_cnt =
(tile_n + 7) /
8; // Possible to have non-1 n_iter_cnt for ab_swap m16 case.
static_assert(m_iter_cnt == 1);
static_assert(m_iter_cnt == 1 || m_iter_cnt == 2);
static_assert(n_iter_cnt == 1 || n_iter_cnt == 2);
__device__ MmaComputer(bf16_t* gmem_c_local_, bf16_t* smem_a_,
@@ -417,12 +417,15 @@ struct MmaComputer {
__device__ void prepare() {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
#pragma unroll
for (int i = 0; i < k_phase_cnt; i++) {
int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256;
int m_idx = linear_idx % tile_m;
int k_idx = linear_idx / tile_m + warp_k_offset_in_tile_k;
k_idx = apply_swizzle_343_on_elem_row_col<bf16_t>(m_idx, k_idx);
a_smem_offsets[0][i] = m_idx * tile_k + k_idx;
for (int m = 0; m < m_iter_cnt; m++) {
#pragma unroll
for (int i = 0; i < k_phase_cnt; i++) {
int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256;
int m_idx = linear_idx % 16 + m * 16;
int k_idx = linear_idx / 16 + warp_k_offset_in_tile_k;
k_idx = apply_swizzle_343_on_elem_row_col<bf16_t>(m_idx, k_idx);
a_smem_offsets[m][i] = m_idx * tile_k + k_idx;
}
}
#pragma unroll
for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) {
@@ -446,11 +449,14 @@ struct MmaComputer {
wait_barrier(smem_barrier + 0 + stage_idx * 2, phase_bit);
#pragma unroll
for (int i = 0; i < k_phase_cnt; i++) {
int smem_offset = a_smem_offsets[0][i];
bf16_t* smem_ptr_this_iter =
smem_a + stage_idx * tile_m * tile_k + smem_offset;
ldsm_x4(smem_ptr_this_iter, reinterpret_cast<uint32_t*>(a_reg[0][i]));
for (int m = 0; m < m_iter_cnt; m++) {
#pragma unroll
for (int i = 0; i < k_phase_cnt; i++) {
int smem_offset = a_smem_offsets[m][i];
bf16_t* smem_ptr_this_iter =
smem_a + stage_idx * tile_m * tile_k + smem_offset;
ldsm_x4(smem_ptr_this_iter, reinterpret_cast<uint32_t*>(a_reg[m][i]));
}
}
#pragma unroll
@@ -469,9 +475,12 @@ struct MmaComputer {
for (int k_iter_idx = 0; k_iter_idx < k_phase_cnt; k_iter_idx++) {
#pragma unroll
for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) {
hmma_16_8_16_f32acc_bf16ab(
acc_reg[0][n_iter_idx], a_reg[0][k_iter_idx],
b_reg[n_iter_idx][k_iter_idx], acc_reg[0][n_iter_idx]);
#pragma unroll
for (int m = 0; m < m_iter_cnt; m++) {
hmma_16_8_16_f32acc_bf16ab(
acc_reg[m][n_iter_idx], a_reg[m][k_iter_idx],
b_reg[n_iter_idx][k_iter_idx], acc_reg[m][n_iter_idx]);
}
}
}
::arrive_barrier(smem_barrier + 1 + stage_idx * 2);
@@ -486,14 +495,14 @@ struct MmaComputer {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
asm volatile("bar.sync %0, %1;" : : "r"(1), "r"(thread_cnt));
// reorganize the acc_reg
constexpr int thread_m = 2;
constexpr int thread_m = 2 * m_iter_cnt;
constexpr int thread_n = 2 * n_iter_cnt;
constexpr int cta_mma_n = n_iter_cnt * 8;
float acc_reg_reorg[thread_m][thread_n];
for (int i = 0; i < thread_m; i++) {
for (int j = 0; j < thread_n; j++) {
acc_reg_reorg[i][j] = acc_reg[0][j / 2][(j % 2) + (i * 2)];
acc_reg_reorg[i][j] = acc_reg[i / 2][j / 2][(j % 2) + (i % 2) * 2];
}
}
@@ -514,7 +523,8 @@ struct MmaComputer {
for (int m_idx_thread = 0; m_idx_thread < thread_m; m_idx_thread++) {
#pragma unroll
for (int n_idx_thread = 0; n_idx_thread < thread_n; n_idx_thread++) {
int m_idx = (lane_idx / 4) + m_idx_thread * 8;
int m_idx =
(lane_idx / 4) + (m_idx_thread % 2) * 8 + (m_idx_thread / 2) * 16;
int n_idx =
((lane_idx % 4) * 2) + (n_idx_thread % 2) + (n_idx_thread / 2) * 8;
smem_c[cosize_smem_c * warp_idx + smem_c_index_func(m_idx, n_idx)] =
@@ -587,7 +597,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
static_assert(
tile_k == 128 || tile_k == 256 || tile_k == 512 ||
tile_k == 1024); // tile_k must be larger than 64 since 4 warp splitK.
static_assert(tile_m == 16);
static_assert(tile_m == 16 || tile_m == 32);
constexpr int g2s_vec_bytes = 16;
constexpr int a_elem_bytes = 2;
constexpr int b_elem_bytes = 2;
@@ -647,7 +657,8 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
#endif
}
template <typename T, int kHdIn, int kHdOut, int kTileN, int kTileK = 256>
template <typename T, int kHdIn, int kHdOut, int kTileN, int kTileK = 256,
int kTileM = 16>
void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
cudaStream_t const stream, bool enable_pdl) {
constexpr int gemm_m = kHdOut;
@@ -655,7 +666,7 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
constexpr int gemm_k = kHdIn;
constexpr int batch_size = 1;
std::swap(mat_a, mat_b);
constexpr int tile_m = 16;
constexpr int tile_m = kTileM;
constexpr int tile_n = kTileN;
constexpr int tile_k = kTileK;
constexpr int max_stage_cnt =
@@ -695,15 +706,15 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
output, mat_a, mat_b, gemm_n);
}
template <typename T, int kHdIn, int kHdOut, int kTileK = 256>
template <typename T, int kHdIn, int kHdOut, int kTileK = 256, int kTileM = 16>
void invokeFusedAGemmForTokens(T* output, T const* mat_a, T const* mat_b,
int num_tokens, cudaStream_t const stream,
bool enable_pdl) {
if (num_tokens <= 8) {
invokeFusedAGemm<T, kHdIn, kHdOut, 8, kTileK>(
invokeFusedAGemm<T, kHdIn, kHdOut, 8, kTileK, kTileM>(
output, mat_a, mat_b, num_tokens, stream, enable_pdl);
} else {
invokeFusedAGemm<T, kHdIn, kHdOut, 16, kTileK>(
invokeFusedAGemm<T, kHdIn, kHdOut, 16, kTileK, kTileM>(
output, mat_a, mat_b, num_tokens, stream, enable_pdl);
}
}
@@ -782,6 +793,13 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output,
DISPATCH_DSV3_SHAPE(7168, 3216)
DISPATCH_DSV3_SHAPE(7168, 4224)
if (hd_in == 6144 && hd_out == 2624) {
invokeFusedAGemmForTokens<__nv_bfloat16, 6144, 2624, 256, 32>(
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl);
return;
}
DISPATCH_DSV3_SHAPE(2048, 2048)
#ifdef VLLM_K3_BENCH_SHAPES
// The selector routes these shapes to CuTe or the default GEMM, so they are
// never reached in production. They are compiled only for offline
+255 -6
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the Kimi-K3 SM103 decode GEMM selector (shape-only dispatch)."""
"""Tests for BF16 skinny GEMMs and the Kimi-K3 SM103 selector."""
from pathlib import Path
from types import SimpleNamespace
@@ -11,7 +11,10 @@ import regex as re
import torch
from torch import nn
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import SkinnyGemmConfig
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
SkinnyGemmConfig,
)
from vllm.models.deepseek_v32.nvidia import glm52_low_latency_gemm as glm52_gemm
from vllm.models.kimi_k3.nvidia import low_latency_gemm as k3_gemm
from vllm.models.kimi_k3.nvidia.low_latency_gemm import KIMI_K3_PROJECTIONS
@@ -60,6 +63,12 @@ RESIDUAL_CUTE_CASES = [
for num_tokens, _ in spec.residual_configs
]
GLM_CUTE_CASES = [
(spec, config)
for spec in glm52_gemm.GLM52_PROJECTIONS.values()
for _, config in spec.cute_configs
]
EXPECTED_CUTE_CONFIGS = {
(3072, 7168, 1): (224, 3, 4, 8),
(3072, 7168, 2): (128, 3, 2, 8),
@@ -164,13 +173,21 @@ def test_every_dsv3_routed_shape_is_instantiated() -> None:
)
} | {
(int(hd_in), int(hd_out))
for hd_in, hd_out in re.findall(r"hd_in == (\d+) && hd_out == (\d+)", explicit)
for hd_in, hd_out in re.findall(
r"hd_in == (\d+) && hd_out == (\d+)",
production_macros + explicit,
)
}
assert compiled, "failed to parse the dispatch list"
specs = [
*KIMI_K3_PROJECTIONS.values(),
glm52_gemm.GLM52_QKV_A_PROJECTION,
glm52_gemm.GLM52_Q_B_PROJECTION,
]
missing = sorted(
(spec.n, spec.k)
for spec in KIMI_K3_PROJECTIONS.values()
for spec in specs
if spec.dsv3_tokens and (spec.k, spec.n) not in compiled
)
assert not missing, (
@@ -190,12 +207,17 @@ def test_packed_row_major_rejects_single_row_slice() -> None:
def test_cute_configs_match_measured_table() -> None:
actual = {
(spec.n, spec.k, num_tokens): _config_tuple(config)
configs = [
(spec.n, spec.k, num_tokens, config)
for spec in k3_gemm.KIMI_K3_PROJECTIONS.values()
for num_tokens, config in spec.cute_configs
]
actual = {
(n, k, num_tokens): _config_tuple(config)
for n, k, num_tokens, config in configs
}
assert actual == EXPECTED_CUTE_CONFIGS
assert all(config.static_k is None for *_, config in configs)
def test_residual_cute_configs_match_measured_table() -> None:
@@ -207,6 +229,134 @@ def test_residual_cute_configs_match_measured_table() -> None:
assert actual == EXPECTED_RESIDUAL_CUTE_CONFIGS
def test_glm52_projection_plans_are_separate() -> None:
qkv_a = glm52_gemm.GLM52_QKV_A_PROJECTION
q_b = glm52_gemm.GLM52_Q_B_PROJECTION
qkv_a_plan = qkv_a.build_plan()
q_b_plan = q_b.build_plan()
assert (qkv_a.n, qkv_a.k, set(qkv_a_plan)) == (
2624,
6144,
set(range(1, 17)),
)
assert (q_b.n, q_b.k, set(q_b_plan)) == (
2048,
2048,
set(range(1, 17)),
)
assert {
num_tokens
for num_tokens, (backend, _) in qkv_a_plan.items()
if backend == "cute"
} == {1, 2}
assert {
num_tokens
for num_tokens, (backend, _) in qkv_a_plan.items()
if backend == "dsv3_fused_a"
} == set(range(3, 17))
assert {
num_tokens for num_tokens, (backend, _) in q_b_plan.items() if backend == "cute"
} == {1, 2}
assert {
num_tokens
for num_tokens, (backend, _) in q_b_plan.items()
if backend == "dsv3_fused_a"
} == set(range(3, 17))
eh = glm52_gemm.GLM52_EH_PROJECTION
eh_plan = eh.build_plan()
assert (eh.n, eh.k) == (6144, 12288)
# The MTP eh_proj has no dsv3 winners; M >= 4 falls back to cuBLAS.
assert set(eh_plan) == {1, 2, 3}
assert all(backend == "cute" for backend, _ in eh_plan.values())
def test_glm52_layout_rejects_nonpacked_single_row_view() -> None:
single_row = torch.empty(1, 144)[:, :128]
multiple_rows = torch.empty(2, 144)[:, :128]
assert single_row.stride() == multiple_rows.stride() == (144, 1)
assert not glm52_gemm._is_supported_row_major(single_row)
assert not glm52_gemm._is_supported_row_major(multiple_rows)
def test_glm52_installer_maps_only_selected_unquantized_shapes(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class FakeLinearBase(nn.Module):
def __init__(
self,
n: int,
k: int,
quant_method: object,
) -> None:
super().__init__()
self.weight = nn.Parameter(
torch.empty(
n,
k,
dtype=torch.bfloat16,
device="meta",
)
)
self.quant_method = quant_method
qkv_a = glm52_gemm.GLM52_QKV_A_PROJECTION
q_b = glm52_gemm.GLM52_Q_B_PROJECTION
root = nn.Module()
root.attn = nn.Module()
root.attn.fused_qkv_a_proj = FakeLinearBase(
qkv_a.n, qkv_a.k, glm52_gemm.UnquantizedLinearMethod()
)
root.attn.q_b_proj = FakeLinearBase(
q_b.n, q_b.k, glm52_gemm.UnquantizedLinearMethod()
)
root.same_shape_other_name = FakeLinearBase(
qkv_a.n, qkv_a.k, glm52_gemm.UnquantizedLinearMethod()
)
root.quantized = nn.Module()
quantized_method = object()
root.quantized.q_b_proj = FakeLinearBase(q_b.n, q_b.k, quantized_method)
root.wrong_shape = nn.Module()
root.wrong_shape.fused_qkv_a_proj = FakeLinearBase(
qkv_a.n + 1, qkv_a.k, glm52_gemm.UnquantizedLinearMethod()
)
monkeypatch.setattr(glm52_gemm, "LinearBase", FakeLinearBase)
monkeypatch.setattr(glm52_gemm, "_is_sm103", lambda: True)
monkeypatch.setattr(
glm52_gemm.shape_dynamic_skinny_gemm,
"is_available",
lambda: False,
)
glm52_gemm.enable_glm52_low_latency_gemm(
root,
torch.bfloat16,
)
assert isinstance(
root.attn.fused_qkv_a_proj.quant_method,
glm52_gemm.GLM52LowLatencyLinearMethod,
)
assert isinstance(
root.attn.q_b_proj.quant_method,
glm52_gemm.GLM52LowLatencyLinearMethod,
)
assert root.attn.fused_qkv_a_proj.quant_method._plan == qkv_a.build_plan()
assert root.attn.q_b_proj.quant_method._plan == q_b.build_plan()
assert isinstance(
root.same_shape_other_name.quant_method,
glm52_gemm.GLM52LowLatencyLinearMethod,
)
assert root.quantized.q_b_proj.quant_method is quantized_method
assert (
type(root.wrong_shape.fused_qkv_a_proj.quant_method)
is glm52_gemm.UnquantizedLinearMethod
)
@pytest.mark.parametrize("key", EXPECTED_SELECTIONS)
def test_sm103_selector_table(key: tuple[int, int]) -> None:
n, k = key
@@ -361,6 +511,76 @@ def _require_sm103_and_cute() -> None:
pytest.skip("CuTe DSL is not available")
@pytest.mark.parametrize("spec,config", GLM_CUTE_CASES)
def test_glm_cute_selected_shapes(
spec: glm52_gemm.GLM52ProjectionSpec,
config: SkinnyGemmConfig,
) -> None:
_require_sm103_and_cute()
torch.manual_seed(42)
x = torch.randn(
config.num_rows,
spec.k,
dtype=torch.bfloat16,
device="cuda",
)
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
plan = spec.build_plan()
output = glm52_gemm.run_glm52_plan(plan, x, weight)
assert output is not None
reference = x.float() @ weight.float().t()
torch.testing.assert_close(output.float(), reference, rtol=2e-2, atol=2e-1)
def test_glm52_q_b_nonpacked_single_row_falls_back() -> None:
_require_sm103_and_cute()
spec = glm52_gemm.GLM52_Q_B_PROJECTION
storage = torch.randn(
1,
glm52_gemm.GLM52_QKV_A_PROJECTION.n,
dtype=torch.bfloat16,
device="cuda",
)
x = storage[:, : spec.k]
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
assert x.stride() == (glm52_gemm.GLM52_QKV_A_PROJECTION.n, 1)
assert not glm52_gemm._runtime_ok(x, weight)
output = glm52_gemm.run_glm52_plan(spec.build_plan(), x, weight)
assert output is None
@pytest.mark.parametrize("spec,config", GLM_CUTE_CASES)
def test_glm_cute_selected_shapes_cuda_graph_capture(
spec: glm52_gemm.GLM52ProjectionSpec,
config: SkinnyGemmConfig,
) -> None:
_require_sm103_and_cute()
x = torch.randn(
config.num_rows,
spec.k,
dtype=torch.bfloat16,
device="cuda",
)
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
plan = spec.build_plan()
glm52_gemm.run_glm52_plan(plan, x, weight)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = glm52_gemm.run_glm52_plan(plan, x, weight)
graph.replay()
torch.accelerator.synchronize()
assert output is not None
reference = x.float() @ weight.float().t()
torch.testing.assert_close(output.float(), reference, rtol=2e-2, atol=2e-1)
@pytest.mark.parametrize("n,k,num_tokens", CUTE_CASES)
def test_cute_selected_shapes(n: int, k: int, num_tokens: int) -> None:
_require_sm103_and_cute()
@@ -393,6 +613,15 @@ DSV3_CASES = sorted(
for num_tokens in _dsv3_probe_tokens(spec.dsv3_tokens)
)
GLM_DSV3_CASES = [
(num_tokens, spec)
for spec in (
glm52_gemm.GLM52_QKV_A_PROJECTION,
glm52_gemm.GLM52_Q_B_PROJECTION,
)
for num_tokens in sorted(_dsv3_probe_tokens(spec.dsv3_tokens))
]
@pytest.mark.parametrize("num_tokens,n,k", DSV3_CASES)
def test_dsv3_selected_shapes(num_tokens: int, n: int, k: int) -> None:
@@ -413,6 +642,26 @@ def test_dsv3_selected_shapes(num_tokens: int, n: int, k: int) -> None:
assert cosine > 0.999
@pytest.mark.parametrize("num_tokens,spec", GLM_DSV3_CASES)
def test_glm_dsv3_selected_shapes(
num_tokens: int,
spec: glm52_gemm.GLM52ProjectionSpec,
) -> None:
_require_sm103_and_dsv3()
torch.manual_seed(42)
x = torch.randn(num_tokens, spec.k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda")
output = glm52_gemm.run_glm52_plan(spec.build_plan(), x, weight)
assert output is not None
reference = torch.nn.functional.linear(x, weight)
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.float().flatten(), dim=0
).item()
assert cosine > 0.999
def test_nonpacked_single_token_dsv3_falls_back() -> None:
_require_sm103_and_dsv3()
n, k = 1536, 128
@@ -8,6 +8,8 @@ import cutlass.cute as cute
from cuda.bindings.driver import CUstream
from cutlass import const_expr
_PREFETCH_TILES = 2
class CuteSkinnyGemm:
"""Shape-dynamic low-latency GEMM for small token counts.
@@ -15,8 +17,9 @@ class CuteSkinnyGemm:
Computes ``C[M, N] = A[M, K] @ B[N, K].T + residual`` with BF16 or FP16
inputs, FP32 accumulators, and an output matching the input dtype. The
residual term is optional and is added before the output conversion. N and
K are runtime values. The tiny M dimension is fully unrolled alongside a
small set of tuning parameters.
K are runtime values unless the configuration specializes K at compile
time. The tiny M dimension is fully unrolled alongside a small set of
tuning parameters.
"""
def __init__(
@@ -30,9 +33,15 @@ class CuteSkinnyGemm:
k_unroll: int = 1,
has_residual: bool = False,
use_pdl: bool = False,
static_k: int | None = None,
) -> None:
if block_size % cute.arch.WARP_SIZE != 0:
raise ValueError("block_size must be a multiple of the warp size")
k_tile_size = block_size * vector_width
if static_k is not None and (
static_k % k_tile_size or static_k < _PREFETCH_TILES * k_tile_size
):
raise ValueError("static K must contain at least two complete tiles")
self.element_type = element_type
self.num_rows = num_rows
self.block_size = block_size
@@ -41,6 +50,7 @@ class CuteSkinnyGemm:
self.k_unroll = k_unroll
self.has_residual = has_residual
self.use_pdl = use_pdl
self.static_k = static_k
self.num_warps = block_size // cute.arch.WARP_SIZE
@cute.jit
@@ -66,7 +76,22 @@ class CuteSkinnyGemm:
num_bits_per_copy=self.vector_width * self.element_type.width,
load_cache_mode=cute.nvgpu.LoadCacheMode.STREAMING,
)
self.kernel(gA, gB, gResidual, gC, k, copy_a, copy_b).launch(
copy_b_prefetch = cute.make_copy_atom(
cute.nvgpu.CopyG2ROp(),
self.element_type,
num_bits_per_copy=self.vector_width * self.element_type.width,
load_cache_mode=cute.nvgpu.LoadCacheMode.ALWAYS,
)
self.kernel(
gA,
gB,
gResidual,
gC,
k,
copy_a,
copy_b,
copy_b_prefetch,
).launch(
grid=[cute.ceil_div(n, self.outputs_per_block), 1, 1],
block=[self.block_size, 1, 1],
smem=self.num_rows * self.outputs_per_block * self.num_warps * 4,
@@ -85,6 +110,7 @@ class CuteSkinnyGemm:
k_extent: cutlass.Int32,
copy_a: cute.CopyAtom,
copy_b: cute.CopyAtom,
copy_b_prefetch: cute.CopyAtom,
) -> None:
tidx, _, _ = cute.arch.thread_idx()
block_idx, _, _ = cute.arch.block_idx()
@@ -102,12 +128,8 @@ class CuteSkinnyGemm:
acc = cute.make_rmem_tensor(acc_layout, cutlass.Float32)
acc.fill(0.0)
if const_expr(self.use_pdl):
cute.arch.griddepcontrol_wait()
n_base = block_idx * outputs_per_block
k_tile_size: cutlass.Constexpr = block_size * vector_width
num_k_tiles = k_extent // k_tile_size
gA_vec = cute.logical_divide(gA, (None, vector_width))
gB_vec = cute.logical_divide(gB, (None, vector_width))
@@ -127,21 +149,72 @@ class CuteSkinnyGemm:
self.element_type,
)
for k_tile in cutlass.range(num_k_tiles, unroll=self.k_unroll):
for mi in cutlass.range_constexpr(num_rows):
cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None])
if const_expr(self.static_k is not None):
num_k_tiles: cutlass.Constexpr = self.static_k // k_tile_size
prefetched_b = cute.make_rmem_tensor(
cute.make_layout(
(_PREFETCH_TILES, outputs_per_block, vector_width),
stride=(outputs_per_block * vector_width, vector_width, 1),
),
self.element_type,
)
for k_tile in cutlass.range_constexpr(_PREFETCH_TILES):
for ni in cutlass.range_constexpr(outputs_per_block):
tB_static = tB_all[n_base + ni, (None, (tidx, None))]
cute.copy(
copy_b_prefetch,
tB_static[None, k_tile],
prefetched_b[k_tile, ni, None],
)
for ni in cutlass.range_constexpr(outputs_per_block):
n_idx = n_base + ni
tB = tB_all[n_idx, (None, (tidx, None))]
cute.copy(copy_b, tB[None, k_tile], b_regs[ni, None])
if const_expr(self.use_pdl):
cute.arch.griddepcontrol_wait()
for vi in cutlass.range_constexpr(vector_width):
for mi in cutlass.range_constexpr(num_rows):
for k_tile in cutlass.range_constexpr(num_k_tiles):
if const_expr(k_tile < _PREFETCH_TILES):
b_f32 = prefetched_b[k_tile, None, None].load().to(cutlass.Float32)
else:
for ni in cutlass.range_constexpr(outputs_per_block):
acc[mi, ni] = acc[mi, ni] + a_regs[mi, vi].to(
cutlass.Float32
) * b_regs[ni, vi].to(cutlass.Float32)
tB_static = tB_all[n_base + ni, (None, (tidx, None))]
cute.copy(
copy_b,
tB_static[None, k_tile],
b_regs[ni, None],
)
b_f32 = b_regs.load().to(cutlass.Float32)
for mi in cutlass.range_constexpr(num_rows):
cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None])
a_f32 = a_regs.load().to(cutlass.Float32)
for vi in cutlass.range_constexpr(vector_width):
for mi in cutlass.range_constexpr(num_rows):
for ni in cutlass.range_constexpr(outputs_per_block):
acc[mi, ni] += a_f32[mi, vi] * b_f32[ni, vi]
else:
if const_expr(self.use_pdl):
cute.arch.griddepcontrol_wait()
num_k_tiles = k_extent // k_tile_size
for k_tile in cutlass.range(num_k_tiles, unroll=self.k_unroll):
for mi in cutlass.range_constexpr(num_rows):
cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None])
for ni in cutlass.range_constexpr(outputs_per_block):
n_idx = n_base + ni
tB_dynamic = tB_all[n_idx, (None, (tidx, None))]
cute.copy(
copy_b,
tB_dynamic[None, k_tile],
b_regs[ni, None],
)
for vi in cutlass.range_constexpr(vector_width):
for mi in cutlass.range_constexpr(num_rows):
for ni in cutlass.range_constexpr(outputs_per_block):
acc[mi, ni] = acc[mi, ni] + a_regs[mi, vi].to(
cutlass.Float32
) * b_regs[ni, vi].to(cutlass.Float32)
for mi in cutlass.range_constexpr(num_rows):
for ni in cutlass.range_constexpr(outputs_per_block):
@@ -23,6 +23,7 @@ class SkinnyGemmConfig:
outputs_per_block: int
k_unroll: int = 1
vector_width: int = 8
static_k: int | None = None
class ShapeDynamicSkinnyGemm:
@@ -112,7 +113,11 @@ class ShapeDynamicSkinnyGemm:
element_type = self._cutlass_dtype(dtype)
n = cute.sym_int(divisibility=config.outputs_per_block)
k = cute.sym_int(divisibility=config.block_size * config.vector_width)
k = (
config.static_k
if config.static_k is not None
else cute.sym_int(divisibility=config.block_size * config.vector_width)
)
a = make_fake_tensor(
element_type,
(config.num_rows, k),
@@ -134,6 +139,7 @@ class ShapeDynamicSkinnyGemm:
k_unroll=config.k_unroll,
has_residual=has_residual,
use_pdl=self._use_pdl(),
static_k=config.static_k,
)
self._compiled[(dtype, config, has_residual)] = cute.compile(
kernel,
@@ -239,6 +245,8 @@ class ShapeDynamicSkinnyGemm:
raise ValueError(
"K must be divisible by block_size * vector_width for this config"
)
if config.static_k is not None and a.shape[1] != config.static_k:
raise ValueError("input K must match config static_k")
has_residual = residual is not None
cache_key = (a.dtype, config, has_residual)
if cache_key not in self._compiled:
@@ -0,0 +1,199 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GLM-5.2 decode GEMM selection for unquantized BF16 on SM103."""
from __future__ import annotations
from dataclasses import dataclass
from typing import Literal
import torch
from torch import nn
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
SkinnyGemmConfig,
shape_dynamic_skinny_gemm,
)
from vllm.model_executor.layers.linear import (
LinearBase,
UnquantizedLinearMethod,
)
from vllm.platforms import current_platform
Backend = Literal["cute", "dsv3_fused_a"]
ResolvedCall = tuple[Backend, SkinnyGemmConfig | None]
@dataclass(frozen=True, slots=True)
class GLM52ProjectionSpec:
n: int
k: int
cute_configs: tuple[tuple[int, SkinnyGemmConfig], ...]
dsv3_tokens: frozenset[int] = frozenset()
def build_plan(self) -> dict[int, ResolvedCall]:
plan: dict[int, ResolvedCall] = {
num_tokens: ("cute", config) for num_tokens, config in self.cute_configs
}
plan.update(
(num_tokens, ("dsv3_fused_a", None)) for num_tokens in self.dsv3_tokens
)
return plan
GLM52_QKV_A_PROJECTION = GLM52ProjectionSpec(
n=2624,
k=6144,
cute_configs=(
(1, SkinnyGemmConfig(1, 128, 4, static_k=6144)),
(2, SkinnyGemmConfig(2, 128, 2)),
),
dsv3_tokens=frozenset(range(3, 17)),
)
GLM52_Q_B_PROJECTION = GLM52ProjectionSpec(
n=2048,
k=2048,
cute_configs=(
(1, SkinnyGemmConfig(1, 128, 4, static_k=2048)),
(2, SkinnyGemmConfig(2, 64, 2, k_unroll=2)),
),
dsv3_tokens=frozenset(range(3, 17)),
)
# The MTP eh_proj is a plain nn.Linear, so it gets its plan through
# build_glm52_plan rather than a quant_method swap. cuBLAS wins from M=4.
GLM52_EH_PROJECTION = GLM52ProjectionSpec(
n=6144,
k=12288,
cute_configs=(
(1, SkinnyGemmConfig(1, 256, 2, vector_width=4, static_k=12288)),
(2, SkinnyGemmConfig(2, 64, 2)),
(3, SkinnyGemmConfig(3, 64, 2)),
),
)
GLM52_PROJECTIONS = {
(spec.n, spec.k): spec
for spec in (
GLM52_QKV_A_PROJECTION,
GLM52_Q_B_PROJECTION,
GLM52_EH_PROJECTION,
)
}
def _is_sm103() -> bool:
return current_platform.is_device_capability((10, 3))
def _is_supported_row_major(tensor: torch.Tensor) -> bool:
return tensor.dim() == 2 and tensor.stride() == (tensor.shape[1], 1)
def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool:
return (
not envs.VLLM_BATCH_INVARIANT
and _is_supported_row_major(x)
and _is_supported_row_major(weight)
and x.dtype == torch.bfloat16
and weight.dtype == torch.bfloat16
and x.is_cuda
and weight.is_cuda
and x.device == weight.device
and x.shape[1] == weight.shape[1]
)
def run_glm52_plan(
plan: dict[int, ResolvedCall] | None,
x: torch.Tensor,
weight: torch.Tensor,
) -> torch.Tensor | None:
if plan is None or not _runtime_ok(x, weight):
return None
entry = plan.get(x.shape[0])
if entry is None:
return None
backend, config = entry
if backend == "cute":
if not shape_dynamic_skinny_gemm.is_available():
return None
return shape_dynamic_skinny_gemm(x, weight, config)
if not hasattr(torch.ops._C, "dsv3_fused_a_gemm"):
return None
output = torch.empty(
(x.shape[0], weight.shape[0]),
dtype=x.dtype,
device=x.device,
)
ops.dsv3_fused_a_gemm(output, x, weight.t(), enable_pdl=True)
return output
def _request_warmup(dtype: torch.dtype, configs: set[SkinnyGemmConfig]) -> None:
if configs and shape_dynamic_skinny_gemm.is_available():
shape_dynamic_skinny_gemm.request_warmup_configs(dtype, configs)
def build_glm52_plan(
weight: torch.Tensor | None, dtype: torch.dtype
) -> dict[int, ResolvedCall] | None:
"""Plan for a weight the walk below cannot reach (a plain ``nn.Linear``)."""
if dtype != torch.bfloat16 or not _is_sm103():
return None
if weight is None or weight.dim() != 2 or weight.dtype != torch.bfloat16:
return None
spec = GLM52_PROJECTIONS.get(tuple(weight.shape))
if spec is None:
return None
_request_warmup(dtype, {config for _, config in spec.cute_configs})
return spec.build_plan()
class GLM52LowLatencyLinearMethod(UnquantizedLinearMethod):
def __init__(self, plan: dict[int, ResolvedCall]) -> None:
super().__init__()
self._plan = plan
def apply(
self,
layer: nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if bias is None:
output = run_glm52_plan(self._plan, x, layer.weight)
if output is not None:
return output
return super().apply(layer, x, bias)
def enable_glm52_low_latency_gemm(
module: nn.Module,
dtype: torch.dtype,
) -> None:
if dtype != torch.bfloat16 or not _is_sm103():
return
warmup_configs: set[SkinnyGemmConfig] = set()
for child in module.modules():
if (
not isinstance(child, LinearBase)
or type(child.quant_method) is not UnquantizedLinearMethod
):
continue
weight = getattr(child, "weight", None)
if weight is None or weight.dim() != 2 or weight.dtype != torch.bfloat16:
continue
spec = GLM52_PROJECTIONS.get(tuple(weight.shape))
if spec is None:
continue
child.quant_method = GLM52LowLatencyLinearMethod(spec.build_plan())
warmup_configs.update(config for _, config in spec.cute_configs)
_request_warmup(dtype, warmup_configs)
+7
View File
@@ -42,6 +42,8 @@ from vllm.models.common.ops.fused_allreduce_rms_norm import fused_allreduce_rms_
from vllm.models.deepseek_v32.attention import DeepseekV32Attention
from vllm.sequence import IntermediateTensors
from .glm52_low_latency_gemm import enable_glm52_low_latency_gemm
def _all_gather_sp_states(
hidden_states: torch.Tensor,
@@ -405,6 +407,11 @@ class DeepseekV32ForCausalLM(DeepseekV2ForCausalLM):
model_cls = DeepseekV32Model
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
if self.config.model_type == "glm_moe_dsa":
enable_glm52_low_latency_gemm(self, vllm_config.model_config.dtype)
def set_moe_parameters(self):
# Same as the base, but keyed on the MoE block type rather than the
# decoder-layer type (DeepseekV32DecoderLayer is a plain nn.Module).
+15 -1
View File
@@ -42,6 +42,11 @@ from vllm.models.deepseek_v32.common.kernels import fused_eh_norm
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from .glm52_low_latency_gemm import (
build_glm52_plan,
enable_glm52_low_latency_gemm,
run_glm52_plan,
)
from .model import DeepseekV32DecoderLayer
@@ -56,6 +61,11 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
self._eh_plan = (
build_glm52_plan(self.eh_proj.weight, vllm_config.model_config.dtype)
if config.model_type == "glm_moe_dsa"
else None
)
topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
@@ -91,7 +101,9 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
self.hnorm.weight,
self.enorm.variance_epsilon,
)
hidden_states = self.eh_proj(eh_input)
hidden_states = run_glm52_plan(self._eh_plan, eh_input, self.eh_proj.weight)
if hidden_states is None:
hidden_states = self.eh_proj(eh_input)
hidden_states, residual = self.mtp_block(
positions=positions, hidden_states=hidden_states, residual=None
)
@@ -199,6 +211,8 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
self.model = DeepseekV32MultiTokenPredictor(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
if self.config.model_type == "glm_moe_dsa":
enable_glm52_low_latency_gemm(self, vllm_config.model_config.dtype)
self.set_moe_parameters()
def set_moe_parameters(self):