diff --git a/csrc/cpu/sgl-kernels/gemm.h b/csrc/cpu/sgl-kernels/gemm.h index 8e8425cfc9..db7ad884d0 100644 --- a/csrc/cpu/sgl-kernels/gemm.h +++ b/csrc/cpu/sgl-kernels/gemm.h @@ -59,6 +59,11 @@ inline bool can_use_brgemm(int M) { return brgemm_supported() && M > 4; } +template +inline bool can_use_brgemm(int M, int N) { + return brgemm_supported() && (can_use_brgemm(M) || N != block_size_n()); +} + // work around compiler internal error #define BLOCK_K 128 // 4 * TILE_K diff --git a/csrc/cpu/sgl-kernels/moe_fp8.cpp b/csrc/cpu/sgl-kernels/moe_fp8.cpp index 5c707c67aa..b8fb54ac77 100644 --- a/csrc/cpu/sgl-kernels/moe_fp8.cpp +++ b/csrc/cpu/sgl-kernels/moe_fp8.cpp @@ -60,9 +60,6 @@ void fused_experts_fp_kernel_impl( const int64_t stride_e = 2 * N * packed_K; const int64_t stride_n = packed_K; - int64_t avg_M = std::max(int64_t(1), M * topk / E); - const bool use_brgemm = can_use_brgemm(avg_M); - int64_t B_tmp_size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N); // here we only parallel on half of 2N to fuse silu_and_mul with gemm @@ -70,6 +67,7 @@ void fused_experts_fp_kernel_impl( // get local pointers int tid = get_thread_num(); scalar_t* __restrict__ A = A_tmp + tid * BLOCK_M * K; + bool use_brgemm = false; loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { int64_t n_size = std::min(2 * N - nb * BLOCK_N, BLOCK_N); @@ -86,6 +84,11 @@ void fused_experts_fp_kernel_impl( bool do_unpack = (mb == mb0) || (expert_id != pre_expert_id); int64_t m_size = offsets[mb + 1] - offsets[mb]; + if (m_size == 0) { + return; + } + const bool brg = can_use_brgemm(m_size, n_size); + use_brgemm |= brg; if (nb_offset == 0) { // 1.a load A @@ -111,7 +114,7 @@ void fused_experts_fp_kernel_impl( /* lda */ K, /* ldb */ n_size, /* ldc */ 2 * N, - /* brg */ use_brgemm, + /* brg */ brg, /* block_size_K */ block_size_K, /* do_unpack */ do_unpack); }); @@ -152,10 +155,16 @@ void fused_experts_fp_kernel_impl( parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { int tid = get_thread_num(); alignas(64) scalar_t C[BLOCK_M * BLOCK_K]; + bool use_brgemm = false; loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { int64_t m_size = offsets[mb + 1] - offsets[mb]; + if (m_size == 0) { + return; + } int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); + const bool brg = can_use_brgemm(m_size, n_size); + use_brgemm |= brg; // A ptr from ic1 of [M * topk, N] in sorted order // so as to avoid copy A to tmp buffer again @@ -187,7 +196,7 @@ void fused_experts_fp_kernel_impl( /* lda */ IC, /* ldb */ n_size, /* ldc */ BLOCK_N, - /* brg */ use_brgemm, + /* brg */ brg, /* block_size_K */ block_size_K, /* do_unpack */ do_unpack); diff --git a/tests/kernels/moe/test_cpu_quant_fused_moe.py b/tests/kernels/moe/test_cpu_quant_fused_moe.py index e0e0203c6b..61e8aeb7a9 100644 --- a/tests/kernels/moe/test_cpu_quant_fused_moe.py +++ b/tests/kernels/moe/test_cpu_quant_fused_moe.py @@ -31,6 +31,21 @@ def _prepack_experts(w: torch.Tensor) -> torch.Tensor: return torch.ops._C.convert_weight_packed(w) +def _deterministic_expert_routes( + block_sizes: tuple[int, ...], +) -> tuple[torch.Tensor, torch.Tensor]: + """Create top-1 routes with exact per-expert block occupancies.""" + topk_ids = torch.cat( + [ + torch.full((size,), expert, dtype=torch.int32) + for expert, size in enumerate(block_sizes) + ] + ) + topk_ids = topk_ids.view(-1, 1) + topk_weights = torch.ones(topk_ids.shape, dtype=torch.float32) + return topk_weights, topk_ids + + # =========================================================================== # FP8 W8A16 MoE # =========================================================================== @@ -219,6 +234,40 @@ def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed): torch.testing.assert_close(out_inplace, out, atol=0, rtol=0) +def test_w8a16_block_fp8_cpu_fused_moe_small_expert_blocks(): + """Test FP8 BRGEMM at exact and multi-block expert boundaries.""" + set_random_seed(0) + block_sizes = (4, 5, 33) + N, K, E = 128, 128, len(block_sizes) + M = sum(block_sizes) + + a = torch.randn(M, K, dtype=torch.bfloat16) / math.sqrt(K) + w1, w2, w1_s, w2_s = _make_fp8_moe_weights(E, N, K, BLOCK_SIZE) + topk_weight, topk_ids = _deterministic_expert_routes(block_sizes) + + ref_out = ref_w8a16_block_fp8_moe( + a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, BLOCK_SIZE + ) + pw1, pw2 = _prepack_experts(w1), _prepack_experts(w2) + out = ops.fused_experts_cpu( + a, + pw1, + pw2, + topk_weight, + topk_ids, + False, + ops.CPUQuantMethod.FP8_W8A16, + w1_s, + w2_s, + None, + None, + BLOCK_SIZE, + is_vnni=True, + ) + + torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2) + + # =========================================================================== # MXFP4 W4A16 MoE # =========================================================================== @@ -438,6 +487,48 @@ def test_mxfp4_cpu_fused_moe(M, N, K, E, topk, seed): torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2) +def test_mxfp4_cpu_fused_moe_small_expert_blocks(): + """Test MXFP4 BRGEMM at exact and multi-block expert boundaries.""" + set_random_seed(0) + block_sizes = (4, 5, 33) + N, K, E = 128, 128, len(block_sizes) + M = sum(block_sizes) + dtype = torch.bfloat16 + + a = torch.randn(M, K, dtype=dtype) / 10 + w1_bf16 = torch.randn(E, 2 * N, K, dtype=dtype) / 10 + w1q, w1s = MXFP4QuantizeUtil.quantize(w1_bf16) + w1s = w1s.reshape(E, 2 * N, K // 32) + w1dq = MXFP4QuantizeUtil.dequantize(w1q, dtype, w1s) + + w2_bf16 = torch.randn(E, K, N, dtype=dtype) / 10 + w2q, w2s = MXFP4QuantizeUtil.quantize(w2_bf16) + w2s = w2s.reshape(E, K, N // 32) + w2dq = MXFP4QuantizeUtil.dequantize(w2q, dtype, w2s) + + topk_weight, topk_ids = _deterministic_expert_routes(block_sizes) + ref_out = ref_mxfp4_fused_moe(a, w1dq, w2dq, topk_weight, topk_ids, 1) + + pw1, pw1s = _prepack_mxfp4_experts(w1q, w1s) + pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s) + out = ops.fused_experts_cpu( + a, + pw1, + pw2, + topk_weight, + topk_ids, + False, + ops.CPUQuantMethod.MXFP4, + pw1s, + pw2s, + None, + None, + None, + ) + + torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2) + + @pytest.mark.parametrize("M", [1, 32]) @pytest.mark.parametrize("N,K,E,topk", [(128, 128, 4, 2), (64, 64, 4, 2)]) @pytest.mark.parametrize("seed", [0])