diff --git a/csrc/libtorch_stable/dsv3_fused_a_gemm.cu b/csrc/libtorch_stable/dsv3_fused_a_gemm.cu index 30f1b70eca..4250683b91 100644 --- a/csrc/libtorch_stable/dsv3_fused_a_gemm.cu +++ b/csrc/libtorch_stable/dsv3_fused_a_gemm.cu @@ -326,7 +326,7 @@ struct GmemLoaderB { #endif } - __device__ void issue_mainloop() { + __device__ void issue_mainloop(int row_stride) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 cudaGridDependencySynchronize(); #pragma unroll 1 @@ -350,7 +350,7 @@ struct GmemLoaderB { int linear_idx = local_tid * vec_elems + i * thread_cnt * vec_elems; int n_idx = linear_idx / tile_k; int k_idx = linear_idx % tile_k; - int gmem_offset = n_idx * gemm_k + k_project(k_idx); + int gmem_offset = n_idx * row_stride + k_project(k_idx); bf16_t const* gmem_ptr_this_iter = gmem_b + gmem_offset; ldgsts_128(gmem_ptr_this_iter, smem_ptr_this_iter, preds[i]); } @@ -585,7 +585,8 @@ struct MmaComputer { template __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel( - bf16_t* output, bf16_t const* mat_a, bf16_t const* mat_b, int gemm_n) { + bf16_t* output, bf16_t const* mat_a, bf16_t const* mat_b, int gemm_n, + int input_row_stride) { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 constexpr int load_thread_cnt = 128; constexpr int compute_thread_cnt = 128; @@ -621,7 +622,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel( int cta_m_idx = tile_m * blockIdx.x; int cta_n_idx = tile_n * blockIdx.y; bf16_t const* gmem_a_local = mat_a + cta_m_idx * gemm_k; - bf16_t const* gmem_b_local = mat_b + cta_n_idx * gemm_k; + bf16_t const* gmem_b_local = mat_b + cta_n_idx * input_row_stride; bf16_t* gmem_c_local = output + cta_n_idx * gemm_m + cta_m_idx; int warp_idx = __shfl_sync(0xffffffff, threadIdx.x / 32, 0); @@ -645,7 +646,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel( GmemLoaderB b_loader( gmem_b_local, smem_b, smem_barrier, gemm_n); b_loader.prepare(); - b_loader.issue_mainloop(); + b_loader.issue_mainloop(input_row_stride); } else { MmaComputer mma_computer( gmem_c_local, smem_a, smem_b, smem_barrier, warp_idx, gemm_n); @@ -660,7 +661,8 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel( template void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens, - cudaStream_t const stream, bool enable_pdl) { + int input_row_stride, cudaStream_t const stream, + bool enable_pdl) { constexpr int gemm_m = kHdOut; int const gemm_n = num_tokens; constexpr int gemm_k = kHdIn; @@ -703,19 +705,19 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens, cudaLaunchKernelEx(&config, fused_a_gemm_kernel, - output, mat_a, mat_b, gemm_n); + output, mat_a, mat_b, gemm_n, input_row_stride); } template void invokeFusedAGemmForTokens(T* output, T const* mat_a, T const* mat_b, - int num_tokens, cudaStream_t const stream, - bool enable_pdl) { + int num_tokens, int input_row_stride, + cudaStream_t const stream, bool enable_pdl) { if (num_tokens <= 8) { invokeFusedAGemm( - output, mat_a, mat_b, num_tokens, stream, enable_pdl); + output, mat_a, mat_b, num_tokens, input_row_stride, stream, enable_pdl); } else { invokeFusedAGemm( - output, mat_a, mat_b, num_tokens, stream, enable_pdl); + output, mat_a, mat_b, num_tokens, input_row_stride, stream, enable_pdl); } } @@ -740,11 +742,9 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output, mat_a.get_device_index() == output.get_device_index(), "mat_a, mat_b, and output must be on the same device"); - // The kernels index global memory with raw pointers and packed strides, so - // reject any padded or transposed view rather than reading out of bounds. STD_TORCH_CHECK( - mat_a.stride(0) == hd_in && mat_a.stride(1) == 1, - "mat_a must be a packed row-major [num_tokens, hd_in] tensor"); + mat_a.stride(1) == 1 && (num_tokens == 1 || mat_a.stride(0) % 8 == 0), + "mat_a must have unit inner stride and 16-byte aligned rows"); STD_TORCH_CHECK( output.stride(0) == hd_out && output.stride(1) == 1, "output must be a packed row-major [num_tokens, hd_out] tensor"); @@ -770,12 +770,14 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output, reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()); auto const* mat_b_ptr = reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()); + int const input_row_stride = mat_a.stride(0); -#define DISPATCH_DSV3_SHAPE(HD_IN, HD_OUT) \ - if (hd_in == HD_IN && hd_out == HD_OUT) { \ - invokeFusedAGemmForTokens<__nv_bfloat16, HD_IN, HD_OUT>( \ - output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl); \ - return; \ +#define DISPATCH_DSV3_SHAPE(HD_IN, HD_OUT) \ + if (hd_in == HD_IN && hd_out == HD_OUT) { \ + invokeFusedAGemmForTokens<__nv_bfloat16, HD_IN, HD_OUT>( \ + output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, input_row_stride, \ + stream, enable_pdl); \ + return; \ } // Shapes the Kimi-K3 selector routes to dsv3_fused_a (see the dsv3 winners @@ -795,7 +797,8 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output, if (hd_in == 6144 && hd_out == 2624) { invokeFusedAGemmForTokens<__nv_bfloat16, 6144, 2624, 256, 32>( - output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl); + output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, input_row_stride, stream, + enable_pdl); return; } DISPATCH_DSV3_SHAPE(2048, 2048) @@ -821,30 +824,35 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output, if (hd_in == 128 && hd_out == 1536) { invokeFusedAGemmForTokens<__nv_bfloat16, 128, 1536, 128>( - output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl); + output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, input_row_stride, stream, + enable_pdl); return; } if (hd_in == 128 && hd_out == 3072) { invokeFusedAGemmForTokens<__nv_bfloat16, 128, 3072, 128>( - output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl); + output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, input_row_stride, stream, + enable_pdl); return; } // TP16 KDA f_b_proj and shared_expert down_proj. Neither hd_in is a multiple // of 256, so both need the 128 tile_k. if (hd_in == 128 && hd_out == 768) { invokeFusedAGemmForTokens<__nv_bfloat16, 128, 768, 128>( - output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl); + output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, input_row_stride, stream, + enable_pdl); return; } if (hd_in == 384 && hd_out == 7168) { invokeFusedAGemmForTokens<__nv_bfloat16, 384, 7168, 128>( - output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl); + output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, input_row_stride, stream, + enable_pdl); return; } #ifdef VLLM_K3_BENCH_SHAPES if (hd_in == 4224 && hd_out == 7168) { invokeFusedAGemmForTokens<__nv_bfloat16, 4224, 7168, 128>( - output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl); + output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, input_row_stride, stream, + enable_pdl); return; } #endif diff --git a/tests/kernels/test_bf16_skinny_gemm.py b/tests/kernels/test_bf16_skinny_gemm.py index ad6b3ee9b0..98a49c753c 100644 --- a/tests/kernels/test_bf16_skinny_gemm.py +++ b/tests/kernels/test_bf16_skinny_gemm.py @@ -996,20 +996,20 @@ def test_glm_dsv3_selected_shapes( assert cosine > 0.999 -def test_nonpacked_single_token_dsv3_falls_back() -> None: +@pytest.mark.parametrize("num_tokens", [1, 4]) +def test_kda_f_b_nonpacked_dsv3_dispatches(num_tokens: int) -> None: _require_sm103_and_dsv3() n, k = 1536, 128 - storage = torch.randn(1, k + 16, dtype=torch.bfloat16, device="cuda") - x = storage[:, :k] + storage = torch.randn(num_tokens, 6288, dtype=torch.bfloat16, device="cuda") + x = storage[:, 6144 : 6144 + k] weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda") spec = k3_gemm.KIMI_K3_PROJECTIONS[(n, k)] method = k3_gemm.KimiK3LowLatencyLinearMethod( k3_gemm._build_plan(spec), k3_gemm._build_residual_plan(spec) ) - assert x.is_contiguous() - assert x.stride() == (k + 16, 1) - assert not k3_gemm._runtime_ok(x, weight) # strict guard rejects the view + assert x.stride() == (6288, 1) + assert k3_gemm._runtime_ok(x, weight) output = method.apply(SimpleNamespace(weight=weight), x) reference = torch.nn.functional.linear(x, weight) diff --git a/vllm/models/kimi_k3/nvidia/low_latency_gemm.py b/vllm/models/kimi_k3/nvidia/low_latency_gemm.py index 2b23a8aec1..b3a4853bd1 100644 --- a/vllm/models/kimi_k3/nvidia/low_latency_gemm.py +++ b/vllm/models/kimi_k3/nvidia/low_latency_gemm.py @@ -663,7 +663,9 @@ def _is_packed_row_major(tensor: torch.Tensor) -> bool: def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool: return ( - _is_packed_row_major(x) + x.dim() == 2 + and x.stride(1) == 1 + and (x.shape[0] == 1 or x.stride(0) % 8 == 0) and _is_packed_row_major(weight) and x.dtype == torch.bfloat16 and weight.dtype == torch.bfloat16 @@ -676,7 +678,8 @@ def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool: def _residual_ok(x: torch.Tensor, weight: torch.Tensor, residual: torch.Tensor) -> bool: return ( - residual.dim() == 2 + _is_packed_row_major(x) + and residual.dim() == 2 and residual.dtype == torch.bfloat16 and residual.is_cuda and residual.device == x.device @@ -693,6 +696,8 @@ def _run_plan( return None backend, config = entry if backend == "cute": + if x.shape[0] != 1 and not _is_packed_row_major(x): + return None if not shape_dynamic_skinny_gemm.is_available(): return None return shape_dynamic_skinny_gemm(x, weight, config, None)