diff --git a/csrc/libtorch_stable/dsv3_fused_a_gemm.cu b/csrc/libtorch_stable/dsv3_fused_a_gemm.cu index d87b034be8..30f1b70eca 100644 --- a/csrc/libtorch_stable/dsv3_fused_a_gemm.cu +++ b/csrc/libtorch_stable/dsv3_fused_a_gemm.cu @@ -391,7 +391,7 @@ struct MmaComputer { static constexpr int n_iter_cnt = (tile_n + 7) / 8; // Possible to have non-1 n_iter_cnt for ab_swap m16 case. - static_assert(m_iter_cnt == 1); + static_assert(m_iter_cnt == 1 || m_iter_cnt == 2); static_assert(n_iter_cnt == 1 || n_iter_cnt == 2); __device__ MmaComputer(bf16_t* gmem_c_local_, bf16_t* smem_a_, @@ -417,12 +417,15 @@ struct MmaComputer { __device__ void prepare() { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 #pragma unroll - for (int i = 0; i < k_phase_cnt; i++) { - int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256; - int m_idx = linear_idx % tile_m; - int k_idx = linear_idx / tile_m + warp_k_offset_in_tile_k; - k_idx = apply_swizzle_343_on_elem_row_col(m_idx, k_idx); - a_smem_offsets[0][i] = m_idx * tile_k + k_idx; + for (int m = 0; m < m_iter_cnt; m++) { + #pragma unroll + for (int i = 0; i < k_phase_cnt; i++) { + int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256; + int m_idx = linear_idx % 16 + m * 16; + int k_idx = linear_idx / 16 + warp_k_offset_in_tile_k; + k_idx = apply_swizzle_343_on_elem_row_col(m_idx, k_idx); + a_smem_offsets[m][i] = m_idx * tile_k + k_idx; + } } #pragma unroll for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) { @@ -446,11 +449,14 @@ struct MmaComputer { wait_barrier(smem_barrier + 0 + stage_idx * 2, phase_bit); #pragma unroll - for (int i = 0; i < k_phase_cnt; i++) { - int smem_offset = a_smem_offsets[0][i]; - bf16_t* smem_ptr_this_iter = - smem_a + stage_idx * tile_m * tile_k + smem_offset; - ldsm_x4(smem_ptr_this_iter, reinterpret_cast(a_reg[0][i])); + for (int m = 0; m < m_iter_cnt; m++) { + #pragma unroll + for (int i = 0; i < k_phase_cnt; i++) { + int smem_offset = a_smem_offsets[m][i]; + bf16_t* smem_ptr_this_iter = + smem_a + stage_idx * tile_m * tile_k + smem_offset; + ldsm_x4(smem_ptr_this_iter, reinterpret_cast(a_reg[m][i])); + } } #pragma unroll @@ -469,9 +475,12 @@ struct MmaComputer { for (int k_iter_idx = 0; k_iter_idx < k_phase_cnt; k_iter_idx++) { #pragma unroll for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) { - hmma_16_8_16_f32acc_bf16ab( - acc_reg[0][n_iter_idx], a_reg[0][k_iter_idx], - b_reg[n_iter_idx][k_iter_idx], acc_reg[0][n_iter_idx]); + #pragma unroll + for (int m = 0; m < m_iter_cnt; m++) { + hmma_16_8_16_f32acc_bf16ab( + acc_reg[m][n_iter_idx], a_reg[m][k_iter_idx], + b_reg[n_iter_idx][k_iter_idx], acc_reg[m][n_iter_idx]); + } } } ::arrive_barrier(smem_barrier + 1 + stage_idx * 2); @@ -486,14 +495,14 @@ struct MmaComputer { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 asm volatile("bar.sync %0, %1;" : : "r"(1), "r"(thread_cnt)); // reorganize the acc_reg - constexpr int thread_m = 2; + constexpr int thread_m = 2 * m_iter_cnt; constexpr int thread_n = 2 * n_iter_cnt; constexpr int cta_mma_n = n_iter_cnt * 8; float acc_reg_reorg[thread_m][thread_n]; for (int i = 0; i < thread_m; i++) { for (int j = 0; j < thread_n; j++) { - acc_reg_reorg[i][j] = acc_reg[0][j / 2][(j % 2) + (i * 2)]; + acc_reg_reorg[i][j] = acc_reg[i / 2][j / 2][(j % 2) + (i % 2) * 2]; } } @@ -514,7 +523,8 @@ struct MmaComputer { for (int m_idx_thread = 0; m_idx_thread < thread_m; m_idx_thread++) { #pragma unroll for (int n_idx_thread = 0; n_idx_thread < thread_n; n_idx_thread++) { - int m_idx = (lane_idx / 4) + m_idx_thread * 8; + int m_idx = + (lane_idx / 4) + (m_idx_thread % 2) * 8 + (m_idx_thread / 2) * 16; int n_idx = ((lane_idx % 4) * 2) + (n_idx_thread % 2) + (n_idx_thread / 2) * 8; smem_c[cosize_smem_c * warp_idx + smem_c_index_func(m_idx, n_idx)] = @@ -587,7 +597,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel( static_assert( tile_k == 128 || tile_k == 256 || tile_k == 512 || tile_k == 1024); // tile_k must be larger than 64 since 4 warp splitK. - static_assert(tile_m == 16); + static_assert(tile_m == 16 || tile_m == 32); constexpr int g2s_vec_bytes = 16; constexpr int a_elem_bytes = 2; constexpr int b_elem_bytes = 2; @@ -647,7 +657,8 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel( #endif } -template +template void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens, cudaStream_t const stream, bool enable_pdl) { constexpr int gemm_m = kHdOut; @@ -655,7 +666,7 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens, constexpr int gemm_k = kHdIn; constexpr int batch_size = 1; std::swap(mat_a, mat_b); - constexpr int tile_m = 16; + constexpr int tile_m = kTileM; constexpr int tile_n = kTileN; constexpr int tile_k = kTileK; constexpr int max_stage_cnt = @@ -695,15 +706,15 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens, output, mat_a, mat_b, gemm_n); } -template +template void invokeFusedAGemmForTokens(T* output, T const* mat_a, T const* mat_b, int num_tokens, cudaStream_t const stream, bool enable_pdl) { if (num_tokens <= 8) { - invokeFusedAGemm( + invokeFusedAGemm( output, mat_a, mat_b, num_tokens, stream, enable_pdl); } else { - invokeFusedAGemm( + invokeFusedAGemm( output, mat_a, mat_b, num_tokens, stream, enable_pdl); } } @@ -782,6 +793,13 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output, DISPATCH_DSV3_SHAPE(7168, 3216) DISPATCH_DSV3_SHAPE(7168, 4224) + if (hd_in == 6144 && hd_out == 2624) { + invokeFusedAGemmForTokens<__nv_bfloat16, 6144, 2624, 256, 32>( + output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl); + return; + } + DISPATCH_DSV3_SHAPE(2048, 2048) + #ifdef VLLM_K3_BENCH_SHAPES // The selector routes these shapes to CuTe or the default GEMM, so they are // never reached in production. They are compiled only for offline diff --git a/tests/kernels/test_bf16_skinny_gemm.py b/tests/kernels/test_bf16_skinny_gemm.py index 0d5813dd50..09fb87d8d8 100644 --- a/tests/kernels/test_bf16_skinny_gemm.py +++ b/tests/kernels/test_bf16_skinny_gemm.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for the Kimi-K3 SM103 decode GEMM selector (shape-only dispatch).""" +"""Tests for BF16 skinny GEMMs and the Kimi-K3 SM103 selector.""" from pathlib import Path from types import SimpleNamespace @@ -11,7 +11,10 @@ import regex as re import torch from torch import nn -from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import SkinnyGemmConfig +from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import ( + SkinnyGemmConfig, +) +from vllm.models.deepseek_v32.nvidia import glm52_low_latency_gemm as glm52_gemm from vllm.models.kimi_k3.nvidia import low_latency_gemm as k3_gemm from vllm.models.kimi_k3.nvidia.low_latency_gemm import KIMI_K3_PROJECTIONS @@ -60,6 +63,12 @@ RESIDUAL_CUTE_CASES = [ for num_tokens, _ in spec.residual_configs ] +GLM_CUTE_CASES = [ + (spec, config) + for spec in glm52_gemm.GLM52_PROJECTIONS.values() + for _, config in spec.cute_configs +] + EXPECTED_CUTE_CONFIGS = { (3072, 7168, 1): (224, 3, 4, 8), (3072, 7168, 2): (128, 3, 2, 8), @@ -164,13 +173,21 @@ def test_every_dsv3_routed_shape_is_instantiated() -> None: ) } | { (int(hd_in), int(hd_out)) - for hd_in, hd_out in re.findall(r"hd_in == (\d+) && hd_out == (\d+)", explicit) + for hd_in, hd_out in re.findall( + r"hd_in == (\d+) && hd_out == (\d+)", + production_macros + explicit, + ) } assert compiled, "failed to parse the dispatch list" + specs = [ + *KIMI_K3_PROJECTIONS.values(), + glm52_gemm.GLM52_QKV_A_PROJECTION, + glm52_gemm.GLM52_Q_B_PROJECTION, + ] missing = sorted( (spec.n, spec.k) - for spec in KIMI_K3_PROJECTIONS.values() + for spec in specs if spec.dsv3_tokens and (spec.k, spec.n) not in compiled ) assert not missing, ( @@ -190,12 +207,17 @@ def test_packed_row_major_rejects_single_row_slice() -> None: def test_cute_configs_match_measured_table() -> None: - actual = { - (spec.n, spec.k, num_tokens): _config_tuple(config) + configs = [ + (spec.n, spec.k, num_tokens, config) for spec in k3_gemm.KIMI_K3_PROJECTIONS.values() for num_tokens, config in spec.cute_configs + ] + actual = { + (n, k, num_tokens): _config_tuple(config) + for n, k, num_tokens, config in configs } assert actual == EXPECTED_CUTE_CONFIGS + assert all(config.static_k is None for *_, config in configs) def test_residual_cute_configs_match_measured_table() -> None: @@ -207,6 +229,134 @@ def test_residual_cute_configs_match_measured_table() -> None: assert actual == EXPECTED_RESIDUAL_CUTE_CONFIGS +def test_glm52_projection_plans_are_separate() -> None: + qkv_a = glm52_gemm.GLM52_QKV_A_PROJECTION + q_b = glm52_gemm.GLM52_Q_B_PROJECTION + + qkv_a_plan = qkv_a.build_plan() + q_b_plan = q_b.build_plan() + + assert (qkv_a.n, qkv_a.k, set(qkv_a_plan)) == ( + 2624, + 6144, + set(range(1, 17)), + ) + assert (q_b.n, q_b.k, set(q_b_plan)) == ( + 2048, + 2048, + set(range(1, 17)), + ) + assert { + num_tokens + for num_tokens, (backend, _) in qkv_a_plan.items() + if backend == "cute" + } == {1, 2} + assert { + num_tokens + for num_tokens, (backend, _) in qkv_a_plan.items() + if backend == "dsv3_fused_a" + } == set(range(3, 17)) + assert { + num_tokens for num_tokens, (backend, _) in q_b_plan.items() if backend == "cute" + } == {1, 2} + assert { + num_tokens + for num_tokens, (backend, _) in q_b_plan.items() + if backend == "dsv3_fused_a" + } == set(range(3, 17)) + + eh = glm52_gemm.GLM52_EH_PROJECTION + eh_plan = eh.build_plan() + assert (eh.n, eh.k) == (6144, 12288) + # The MTP eh_proj has no dsv3 winners; M >= 4 falls back to cuBLAS. + assert set(eh_plan) == {1, 2, 3} + assert all(backend == "cute" for backend, _ in eh_plan.values()) + + +def test_glm52_layout_rejects_nonpacked_single_row_view() -> None: + single_row = torch.empty(1, 144)[:, :128] + multiple_rows = torch.empty(2, 144)[:, :128] + + assert single_row.stride() == multiple_rows.stride() == (144, 1) + assert not glm52_gemm._is_supported_row_major(single_row) + assert not glm52_gemm._is_supported_row_major(multiple_rows) + + +def test_glm52_installer_maps_only_selected_unquantized_shapes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FakeLinearBase(nn.Module): + def __init__( + self, + n: int, + k: int, + quant_method: object, + ) -> None: + super().__init__() + self.weight = nn.Parameter( + torch.empty( + n, + k, + dtype=torch.bfloat16, + device="meta", + ) + ) + self.quant_method = quant_method + + qkv_a = glm52_gemm.GLM52_QKV_A_PROJECTION + q_b = glm52_gemm.GLM52_Q_B_PROJECTION + root = nn.Module() + root.attn = nn.Module() + root.attn.fused_qkv_a_proj = FakeLinearBase( + qkv_a.n, qkv_a.k, glm52_gemm.UnquantizedLinearMethod() + ) + root.attn.q_b_proj = FakeLinearBase( + q_b.n, q_b.k, glm52_gemm.UnquantizedLinearMethod() + ) + root.same_shape_other_name = FakeLinearBase( + qkv_a.n, qkv_a.k, glm52_gemm.UnquantizedLinearMethod() + ) + root.quantized = nn.Module() + quantized_method = object() + root.quantized.q_b_proj = FakeLinearBase(q_b.n, q_b.k, quantized_method) + root.wrong_shape = nn.Module() + root.wrong_shape.fused_qkv_a_proj = FakeLinearBase( + qkv_a.n + 1, qkv_a.k, glm52_gemm.UnquantizedLinearMethod() + ) + monkeypatch.setattr(glm52_gemm, "LinearBase", FakeLinearBase) + monkeypatch.setattr(glm52_gemm, "_is_sm103", lambda: True) + monkeypatch.setattr( + glm52_gemm.shape_dynamic_skinny_gemm, + "is_available", + lambda: False, + ) + + glm52_gemm.enable_glm52_low_latency_gemm( + root, + torch.bfloat16, + ) + + assert isinstance( + root.attn.fused_qkv_a_proj.quant_method, + glm52_gemm.GLM52LowLatencyLinearMethod, + ) + assert isinstance( + root.attn.q_b_proj.quant_method, + glm52_gemm.GLM52LowLatencyLinearMethod, + ) + assert root.attn.fused_qkv_a_proj.quant_method._plan == qkv_a.build_plan() + assert root.attn.q_b_proj.quant_method._plan == q_b.build_plan() + assert isinstance( + root.same_shape_other_name.quant_method, + glm52_gemm.GLM52LowLatencyLinearMethod, + ) + assert root.quantized.q_b_proj.quant_method is quantized_method + assert ( + type(root.wrong_shape.fused_qkv_a_proj.quant_method) + is glm52_gemm.UnquantizedLinearMethod + ) + + @pytest.mark.parametrize("key", EXPECTED_SELECTIONS) def test_sm103_selector_table(key: tuple[int, int]) -> None: n, k = key @@ -361,6 +511,76 @@ def _require_sm103_and_cute() -> None: pytest.skip("CuTe DSL is not available") +@pytest.mark.parametrize("spec,config", GLM_CUTE_CASES) +def test_glm_cute_selected_shapes( + spec: glm52_gemm.GLM52ProjectionSpec, + config: SkinnyGemmConfig, +) -> None: + _require_sm103_and_cute() + torch.manual_seed(42) + x = torch.randn( + config.num_rows, + spec.k, + dtype=torch.bfloat16, + device="cuda", + ) + weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda") + plan = spec.build_plan() + + output = glm52_gemm.run_glm52_plan(plan, x, weight) + + assert output is not None + reference = x.float() @ weight.float().t() + torch.testing.assert_close(output.float(), reference, rtol=2e-2, atol=2e-1) + + +def test_glm52_q_b_nonpacked_single_row_falls_back() -> None: + _require_sm103_and_cute() + spec = glm52_gemm.GLM52_Q_B_PROJECTION + storage = torch.randn( + 1, + glm52_gemm.GLM52_QKV_A_PROJECTION.n, + dtype=torch.bfloat16, + device="cuda", + ) + x = storage[:, : spec.k] + weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda") + + assert x.stride() == (glm52_gemm.GLM52_QKV_A_PROJECTION.n, 1) + assert not glm52_gemm._runtime_ok(x, weight) + output = glm52_gemm.run_glm52_plan(spec.build_plan(), x, weight) + + assert output is None + + +@pytest.mark.parametrize("spec,config", GLM_CUTE_CASES) +def test_glm_cute_selected_shapes_cuda_graph_capture( + spec: glm52_gemm.GLM52ProjectionSpec, + config: SkinnyGemmConfig, +) -> None: + _require_sm103_and_cute() + x = torch.randn( + config.num_rows, + spec.k, + dtype=torch.bfloat16, + device="cuda", + ) + weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda") + plan = spec.build_plan() + glm52_gemm.run_glm52_plan(plan, x, weight) + torch.accelerator.synchronize() + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + output = glm52_gemm.run_glm52_plan(plan, x, weight) + graph.replay() + torch.accelerator.synchronize() + + assert output is not None + reference = x.float() @ weight.float().t() + torch.testing.assert_close(output.float(), reference, rtol=2e-2, atol=2e-1) + + @pytest.mark.parametrize("n,k,num_tokens", CUTE_CASES) def test_cute_selected_shapes(n: int, k: int, num_tokens: int) -> None: _require_sm103_and_cute() @@ -393,6 +613,15 @@ DSV3_CASES = sorted( for num_tokens in _dsv3_probe_tokens(spec.dsv3_tokens) ) +GLM_DSV3_CASES = [ + (num_tokens, spec) + for spec in ( + glm52_gemm.GLM52_QKV_A_PROJECTION, + glm52_gemm.GLM52_Q_B_PROJECTION, + ) + for num_tokens in sorted(_dsv3_probe_tokens(spec.dsv3_tokens)) +] + @pytest.mark.parametrize("num_tokens,n,k", DSV3_CASES) def test_dsv3_selected_shapes(num_tokens: int, n: int, k: int) -> None: @@ -413,6 +642,26 @@ def test_dsv3_selected_shapes(num_tokens: int, n: int, k: int) -> None: assert cosine > 0.999 +@pytest.mark.parametrize("num_tokens,spec", GLM_DSV3_CASES) +def test_glm_dsv3_selected_shapes( + num_tokens: int, + spec: glm52_gemm.GLM52ProjectionSpec, +) -> None: + _require_sm103_and_dsv3() + torch.manual_seed(42) + x = torch.randn(num_tokens, spec.k, dtype=torch.bfloat16, device="cuda") + weight = torch.randn(spec.n, spec.k, dtype=torch.bfloat16, device="cuda") + + output = glm52_gemm.run_glm52_plan(spec.build_plan(), x, weight) + + assert output is not None + reference = torch.nn.functional.linear(x, weight) + cosine = torch.nn.functional.cosine_similarity( + output.float().flatten(), reference.float().flatten(), dim=0 + ).item() + assert cosine > 0.999 + + def test_nonpacked_single_token_dsv3_falls_back() -> None: _require_sm103_and_dsv3() n, k = 1536, 128 diff --git a/vllm/model_executor/kernels/linear/cute_dsl/_skinny_gemm.py b/vllm/model_executor/kernels/linear/cute_dsl/_skinny_gemm.py index 44964f3e47..5385391aa1 100644 --- a/vllm/model_executor/kernels/linear/cute_dsl/_skinny_gemm.py +++ b/vllm/model_executor/kernels/linear/cute_dsl/_skinny_gemm.py @@ -8,6 +8,8 @@ import cutlass.cute as cute from cuda.bindings.driver import CUstream from cutlass import const_expr +_PREFETCH_TILES = 2 + class CuteSkinnyGemm: """Shape-dynamic low-latency GEMM for small token counts. @@ -15,8 +17,9 @@ class CuteSkinnyGemm: Computes ``C[M, N] = A[M, K] @ B[N, K].T + residual`` with BF16 or FP16 inputs, FP32 accumulators, and an output matching the input dtype. The residual term is optional and is added before the output conversion. N and - K are runtime values. The tiny M dimension is fully unrolled alongside a - small set of tuning parameters. + K are runtime values unless the configuration specializes K at compile + time. The tiny M dimension is fully unrolled alongside a small set of + tuning parameters. """ def __init__( @@ -30,9 +33,15 @@ class CuteSkinnyGemm: k_unroll: int = 1, has_residual: bool = False, use_pdl: bool = False, + static_k: int | None = None, ) -> None: if block_size % cute.arch.WARP_SIZE != 0: raise ValueError("block_size must be a multiple of the warp size") + k_tile_size = block_size * vector_width + if static_k is not None and ( + static_k % k_tile_size or static_k < _PREFETCH_TILES * k_tile_size + ): + raise ValueError("static K must contain at least two complete tiles") self.element_type = element_type self.num_rows = num_rows self.block_size = block_size @@ -41,6 +50,7 @@ class CuteSkinnyGemm: self.k_unroll = k_unroll self.has_residual = has_residual self.use_pdl = use_pdl + self.static_k = static_k self.num_warps = block_size // cute.arch.WARP_SIZE @cute.jit @@ -66,7 +76,22 @@ class CuteSkinnyGemm: num_bits_per_copy=self.vector_width * self.element_type.width, load_cache_mode=cute.nvgpu.LoadCacheMode.STREAMING, ) - self.kernel(gA, gB, gResidual, gC, k, copy_a, copy_b).launch( + copy_b_prefetch = cute.make_copy_atom( + cute.nvgpu.CopyG2ROp(), + self.element_type, + num_bits_per_copy=self.vector_width * self.element_type.width, + load_cache_mode=cute.nvgpu.LoadCacheMode.ALWAYS, + ) + self.kernel( + gA, + gB, + gResidual, + gC, + k, + copy_a, + copy_b, + copy_b_prefetch, + ).launch( grid=[cute.ceil_div(n, self.outputs_per_block), 1, 1], block=[self.block_size, 1, 1], smem=self.num_rows * self.outputs_per_block * self.num_warps * 4, @@ -85,6 +110,7 @@ class CuteSkinnyGemm: k_extent: cutlass.Int32, copy_a: cute.CopyAtom, copy_b: cute.CopyAtom, + copy_b_prefetch: cute.CopyAtom, ) -> None: tidx, _, _ = cute.arch.thread_idx() block_idx, _, _ = cute.arch.block_idx() @@ -102,12 +128,8 @@ class CuteSkinnyGemm: acc = cute.make_rmem_tensor(acc_layout, cutlass.Float32) acc.fill(0.0) - if const_expr(self.use_pdl): - cute.arch.griddepcontrol_wait() - n_base = block_idx * outputs_per_block k_tile_size: cutlass.Constexpr = block_size * vector_width - num_k_tiles = k_extent // k_tile_size gA_vec = cute.logical_divide(gA, (None, vector_width)) gB_vec = cute.logical_divide(gB, (None, vector_width)) @@ -127,21 +149,72 @@ class CuteSkinnyGemm: self.element_type, ) - for k_tile in cutlass.range(num_k_tiles, unroll=self.k_unroll): - for mi in cutlass.range_constexpr(num_rows): - cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None]) + if const_expr(self.static_k is not None): + num_k_tiles: cutlass.Constexpr = self.static_k // k_tile_size + prefetched_b = cute.make_rmem_tensor( + cute.make_layout( + (_PREFETCH_TILES, outputs_per_block, vector_width), + stride=(outputs_per_block * vector_width, vector_width, 1), + ), + self.element_type, + ) + for k_tile in cutlass.range_constexpr(_PREFETCH_TILES): + for ni in cutlass.range_constexpr(outputs_per_block): + tB_static = tB_all[n_base + ni, (None, (tidx, None))] + cute.copy( + copy_b_prefetch, + tB_static[None, k_tile], + prefetched_b[k_tile, ni, None], + ) - for ni in cutlass.range_constexpr(outputs_per_block): - n_idx = n_base + ni - tB = tB_all[n_idx, (None, (tidx, None))] - cute.copy(copy_b, tB[None, k_tile], b_regs[ni, None]) + if const_expr(self.use_pdl): + cute.arch.griddepcontrol_wait() - for vi in cutlass.range_constexpr(vector_width): - for mi in cutlass.range_constexpr(num_rows): + for k_tile in cutlass.range_constexpr(num_k_tiles): + if const_expr(k_tile < _PREFETCH_TILES): + b_f32 = prefetched_b[k_tile, None, None].load().to(cutlass.Float32) + else: for ni in cutlass.range_constexpr(outputs_per_block): - acc[mi, ni] = acc[mi, ni] + a_regs[mi, vi].to( - cutlass.Float32 - ) * b_regs[ni, vi].to(cutlass.Float32) + tB_static = tB_all[n_base + ni, (None, (tidx, None))] + cute.copy( + copy_b, + tB_static[None, k_tile], + b_regs[ni, None], + ) + b_f32 = b_regs.load().to(cutlass.Float32) + + for mi in cutlass.range_constexpr(num_rows): + cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None]) + a_f32 = a_regs.load().to(cutlass.Float32) + + for vi in cutlass.range_constexpr(vector_width): + for mi in cutlass.range_constexpr(num_rows): + for ni in cutlass.range_constexpr(outputs_per_block): + acc[mi, ni] += a_f32[mi, vi] * b_f32[ni, vi] + else: + if const_expr(self.use_pdl): + cute.arch.griddepcontrol_wait() + + num_k_tiles = k_extent // k_tile_size + for k_tile in cutlass.range(num_k_tiles, unroll=self.k_unroll): + for mi in cutlass.range_constexpr(num_rows): + cute.copy(copy_a, tA[mi, None, k_tile], a_regs[mi, None]) + + for ni in cutlass.range_constexpr(outputs_per_block): + n_idx = n_base + ni + tB_dynamic = tB_all[n_idx, (None, (tidx, None))] + cute.copy( + copy_b, + tB_dynamic[None, k_tile], + b_regs[ni, None], + ) + + for vi in cutlass.range_constexpr(vector_width): + for mi in cutlass.range_constexpr(num_rows): + for ni in cutlass.range_constexpr(outputs_per_block): + acc[mi, ni] = acc[mi, ni] + a_regs[mi, vi].to( + cutlass.Float32 + ) * b_regs[ni, vi].to(cutlass.Float32) for mi in cutlass.range_constexpr(num_rows): for ni in cutlass.range_constexpr(outputs_per_block): diff --git a/vllm/model_executor/kernels/linear/cute_dsl/skinny_gemm.py b/vllm/model_executor/kernels/linear/cute_dsl/skinny_gemm.py index e1dd18920b..73d1b2b067 100644 --- a/vllm/model_executor/kernels/linear/cute_dsl/skinny_gemm.py +++ b/vllm/model_executor/kernels/linear/cute_dsl/skinny_gemm.py @@ -23,6 +23,7 @@ class SkinnyGemmConfig: outputs_per_block: int k_unroll: int = 1 vector_width: int = 8 + static_k: int | None = None class ShapeDynamicSkinnyGemm: @@ -112,7 +113,11 @@ class ShapeDynamicSkinnyGemm: element_type = self._cutlass_dtype(dtype) n = cute.sym_int(divisibility=config.outputs_per_block) - k = cute.sym_int(divisibility=config.block_size * config.vector_width) + k = ( + config.static_k + if config.static_k is not None + else cute.sym_int(divisibility=config.block_size * config.vector_width) + ) a = make_fake_tensor( element_type, (config.num_rows, k), @@ -134,6 +139,7 @@ class ShapeDynamicSkinnyGemm: k_unroll=config.k_unroll, has_residual=has_residual, use_pdl=self._use_pdl(), + static_k=config.static_k, ) self._compiled[(dtype, config, has_residual)] = cute.compile( kernel, @@ -239,6 +245,8 @@ class ShapeDynamicSkinnyGemm: raise ValueError( "K must be divisible by block_size * vector_width for this config" ) + if config.static_k is not None and a.shape[1] != config.static_k: + raise ValueError("input K must match config static_k") has_residual = residual is not None cache_key = (a.dtype, config, has_residual) if cache_key not in self._compiled: diff --git a/vllm/models/deepseek_v32/nvidia/glm52_low_latency_gemm.py b/vllm/models/deepseek_v32/nvidia/glm52_low_latency_gemm.py new file mode 100644 index 0000000000..2872aceea8 --- /dev/null +++ b/vllm/models/deepseek_v32/nvidia/glm52_low_latency_gemm.py @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""GLM-5.2 decode GEMM selection for unquantized BF16 on SM103.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +import torch +from torch import nn + +import vllm.envs as envs +from vllm import _custom_ops as ops +from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import ( + SkinnyGemmConfig, + shape_dynamic_skinny_gemm, +) +from vllm.model_executor.layers.linear import ( + LinearBase, + UnquantizedLinearMethod, +) +from vllm.platforms import current_platform + +Backend = Literal["cute", "dsv3_fused_a"] +ResolvedCall = tuple[Backend, SkinnyGemmConfig | None] + + +@dataclass(frozen=True, slots=True) +class GLM52ProjectionSpec: + n: int + k: int + cute_configs: tuple[tuple[int, SkinnyGemmConfig], ...] + dsv3_tokens: frozenset[int] = frozenset() + + def build_plan(self) -> dict[int, ResolvedCall]: + plan: dict[int, ResolvedCall] = { + num_tokens: ("cute", config) for num_tokens, config in self.cute_configs + } + plan.update( + (num_tokens, ("dsv3_fused_a", None)) for num_tokens in self.dsv3_tokens + ) + return plan + + +GLM52_QKV_A_PROJECTION = GLM52ProjectionSpec( + n=2624, + k=6144, + cute_configs=( + (1, SkinnyGemmConfig(1, 128, 4, static_k=6144)), + (2, SkinnyGemmConfig(2, 128, 2)), + ), + dsv3_tokens=frozenset(range(3, 17)), +) + +GLM52_Q_B_PROJECTION = GLM52ProjectionSpec( + n=2048, + k=2048, + cute_configs=( + (1, SkinnyGemmConfig(1, 128, 4, static_k=2048)), + (2, SkinnyGemmConfig(2, 64, 2, k_unroll=2)), + ), + dsv3_tokens=frozenset(range(3, 17)), +) + +# The MTP eh_proj is a plain nn.Linear, so it gets its plan through +# build_glm52_plan rather than a quant_method swap. cuBLAS wins from M=4. +GLM52_EH_PROJECTION = GLM52ProjectionSpec( + n=6144, + k=12288, + cute_configs=( + (1, SkinnyGemmConfig(1, 256, 2, vector_width=4, static_k=12288)), + (2, SkinnyGemmConfig(2, 64, 2)), + (3, SkinnyGemmConfig(3, 64, 2)), + ), +) + +GLM52_PROJECTIONS = { + (spec.n, spec.k): spec + for spec in ( + GLM52_QKV_A_PROJECTION, + GLM52_Q_B_PROJECTION, + GLM52_EH_PROJECTION, + ) +} + + +def _is_sm103() -> bool: + return current_platform.is_device_capability((10, 3)) + + +def _is_supported_row_major(tensor: torch.Tensor) -> bool: + return tensor.dim() == 2 and tensor.stride() == (tensor.shape[1], 1) + + +def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool: + return ( + not envs.VLLM_BATCH_INVARIANT + and _is_supported_row_major(x) + and _is_supported_row_major(weight) + and x.dtype == torch.bfloat16 + and weight.dtype == torch.bfloat16 + and x.is_cuda + and weight.is_cuda + and x.device == weight.device + and x.shape[1] == weight.shape[1] + ) + + +def run_glm52_plan( + plan: dict[int, ResolvedCall] | None, + x: torch.Tensor, + weight: torch.Tensor, +) -> torch.Tensor | None: + if plan is None or not _runtime_ok(x, weight): + return None + entry = plan.get(x.shape[0]) + if entry is None: + return None + + backend, config = entry + if backend == "cute": + if not shape_dynamic_skinny_gemm.is_available(): + return None + return shape_dynamic_skinny_gemm(x, weight, config) + + if not hasattr(torch.ops._C, "dsv3_fused_a_gemm"): + return None + output = torch.empty( + (x.shape[0], weight.shape[0]), + dtype=x.dtype, + device=x.device, + ) + ops.dsv3_fused_a_gemm(output, x, weight.t(), enable_pdl=True) + return output + + +def _request_warmup(dtype: torch.dtype, configs: set[SkinnyGemmConfig]) -> None: + if configs and shape_dynamic_skinny_gemm.is_available(): + shape_dynamic_skinny_gemm.request_warmup_configs(dtype, configs) + + +def build_glm52_plan( + weight: torch.Tensor | None, dtype: torch.dtype +) -> dict[int, ResolvedCall] | None: + """Plan for a weight the walk below cannot reach (a plain ``nn.Linear``).""" + if dtype != torch.bfloat16 or not _is_sm103(): + return None + if weight is None or weight.dim() != 2 or weight.dtype != torch.bfloat16: + return None + spec = GLM52_PROJECTIONS.get(tuple(weight.shape)) + if spec is None: + return None + _request_warmup(dtype, {config for _, config in spec.cute_configs}) + return spec.build_plan() + + +class GLM52LowLatencyLinearMethod(UnquantizedLinearMethod): + def __init__(self, plan: dict[int, ResolvedCall]) -> None: + super().__init__() + self._plan = plan + + def apply( + self, + layer: nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + if bias is None: + output = run_glm52_plan(self._plan, x, layer.weight) + if output is not None: + return output + return super().apply(layer, x, bias) + + +def enable_glm52_low_latency_gemm( + module: nn.Module, + dtype: torch.dtype, +) -> None: + if dtype != torch.bfloat16 or not _is_sm103(): + return + + warmup_configs: set[SkinnyGemmConfig] = set() + for child in module.modules(): + if ( + not isinstance(child, LinearBase) + or type(child.quant_method) is not UnquantizedLinearMethod + ): + continue + weight = getattr(child, "weight", None) + if weight is None or weight.dim() != 2 or weight.dtype != torch.bfloat16: + continue + spec = GLM52_PROJECTIONS.get(tuple(weight.shape)) + if spec is None: + continue + child.quant_method = GLM52LowLatencyLinearMethod(spec.build_plan()) + warmup_configs.update(config for _, config in spec.cute_configs) + + _request_warmup(dtype, warmup_configs) diff --git a/vllm/models/deepseek_v32/nvidia/model.py b/vllm/models/deepseek_v32/nvidia/model.py index 240e97329e..f678e1a62d 100644 --- a/vllm/models/deepseek_v32/nvidia/model.py +++ b/vllm/models/deepseek_v32/nvidia/model.py @@ -42,6 +42,8 @@ from vllm.models.common.ops.fused_allreduce_rms_norm import fused_allreduce_rms_ from vllm.models.deepseek_v32.attention import DeepseekV32Attention from vllm.sequence import IntermediateTensors +from .glm52_low_latency_gemm import enable_glm52_low_latency_gemm + def _all_gather_sp_states( hidden_states: torch.Tensor, @@ -405,6 +407,11 @@ class DeepseekV32ForCausalLM(DeepseekV2ForCausalLM): model_cls = DeepseekV32Model + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__(vllm_config=vllm_config, prefix=prefix) + if self.config.model_type == "glm_moe_dsa": + enable_glm52_low_latency_gemm(self, vllm_config.model_config.dtype) + def set_moe_parameters(self): # Same as the base, but keyed on the MoE block type rather than the # decoder-layer type (DeepseekV32DecoderLayer is a plain nn.Module). diff --git a/vllm/models/deepseek_v32/nvidia/mtp.py b/vllm/models/deepseek_v32/nvidia/mtp.py index 828cdd0208..fd2e426c63 100644 --- a/vllm/models/deepseek_v32/nvidia/mtp.py +++ b/vllm/models/deepseek_v32/nvidia/mtp.py @@ -42,6 +42,11 @@ from vllm.models.deepseek_v32.common.kernels import fused_eh_norm from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors +from .glm52_low_latency_gemm import ( + build_glm52_plan, + enable_glm52_low_latency_gemm, + run_glm52_plan, +) from .model import DeepseekV32DecoderLayer @@ -56,6 +61,11 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) + self._eh_plan = ( + build_glm52_plan(self.eh_proj.weight, vllm_config.model_config.dtype) + if config.model_type == "glm_moe_dsa" + else None + ) topk_indices_buffer = torch.empty( vllm_config.scheduler_config.max_num_batched_tokens, @@ -91,7 +101,9 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): self.hnorm.weight, self.enorm.variance_epsilon, ) - hidden_states = self.eh_proj(eh_input) + hidden_states = run_glm52_plan(self._eh_plan, eh_input, self.eh_proj.weight) + if hidden_states is None: + hidden_states = self.eh_proj(eh_input) hidden_states, residual = self.mtp_block( positions=positions, hidden_states=hidden_states, residual=None ) @@ -199,6 +211,8 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts): self.model = DeepseekV32MultiTokenPredictor( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") ) + if self.config.model_type == "glm_moe_dsa": + enable_glm52_low_latency_gemm(self, vllm_config.model_config.dtype) self.set_moe_parameters() def set_moe_parameters(self):