diff --git a/tests/kernels/moe/test_deepgemm.py b/tests/kernels/moe/test_deepgemm.py index cee0faa592..385cc596ed 100644 --- a/tests/kernels/moe/test_deepgemm.py +++ b/tests/kernels/moe/test_deepgemm.py @@ -158,6 +158,153 @@ def make_block_quant_fp8_weights( return w1, w2, w1_s, w2_s +@pytest.mark.parametrize("num_tokens", [16, 129, 512]) +@pytest.mark.parametrize("with_metadata", [False, True]) +@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels") +def test_block_fp8_padding_on_graph_replay(num_tokens, with_metadata, monkeypatch): + """Both FP8 GEMMs consume only live rows as routing changes on replay.""" + from unittest.mock import Mock + + import vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe as impl + from vllm.utils.deep_gemm import DeepGemmQuantScaleFMT + + if not ( + impl.current_platform.is_device_capability_family(90) + or ( + impl.current_platform.is_device_capability_family(100) + and DeepGemmQuantScaleFMT.from_oracle() == DeepGemmQuantScaleFMT.UE8M0 + ) + ): + pytest.skip("Prefix-sum FP8 dispatch requires SM90 or SM10.x") + + torch.manual_seed(0) + e, n, k, topk = 8, 1024, 512, 2 + x = torch.randn(num_tokens, k, device="cuda", dtype=torch.bfloat16) / 10 + aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True) + w1, w2, s1, s2 = make_block_quant_fp8_weights(e, n, k, BLOCK_SIZE) + quant = fp8_w8a8_moe_quant_config(w1_scale=s1, w2_scale=s2, block_shape=BLOCK_SIZE) + experts = impl.DeepGemmExperts(make_dummy_moe_config(), quant) + ids = torch.zeros(num_tokens, topk, device="cuda", dtype=torch.int64) + weights = torch.full((num_tokens, topk), 0.5, device="cuda") + counts = torch.zeros(e, device="cuda", dtype=torch.int32) + metadata = mk.ExpertTokensMetadata(counts, None) if with_metadata else None + shapes = experts.workspace_shapes( + num_tokens, 2 * n, k, topk, e, e, metadata, MoEActivation.SILU + ) + + def route(mode): + ids[:, 0] = torch.arange(num_tokens, device="cuda") % e + ids[:, 1] = (ids[:, 0] + 1) % e + if mode == 1: + ids.fill_(-1) + ids[:1, 0] = e - 1 + elif mode == 2: + ids.fill_(-1) + counts.copy_(torch.bincount(ids[ids >= 0], minlength=e).int()) + + route(0) + variants = [] + for dense in (True, False): + ws1, ws2, output = [ + torch.empty(shape, device="cuda", dtype=torch.bfloat16) for shape in shapes + ] + + def invoke(output=output, ws1=ws1, ws2=ws2): + experts.apply( + output, + aq, + w1, + w2, + weights, + ids, + MoEActivation.SILU, + e, + None, + scales, + None, + ws1, + ws2, + metadata, + False, + ) + + with monkeypatch.context() as mp: + if dense: + platform = Mock(wraps=impl.current_platform) + platform.is_device_capability_family.return_value = False + mp.setattr(impl, "current_platform", platform) + gemm = impl.m_grouped_fp8_gemm_nt_contiguous + calls: list[int] = [] + + def check_layout(*args, dense=dense, calls=calls, gemm=gemm, **kwargs): + assert kwargs.get("use_psum_layout", False) == (not dense) + calls.append(args[3].numel()) + return gemm(*args, **kwargs) + + mp.setattr(impl, "m_grouped_fp8_gemm_nt_contiguous", check_layout) + invoke() + assert len(calls) == 2 + if not dense: + assert calls == [e, e] + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + invoke() + variants.append((graph, output, ws1, ws2)) + + for mode in (0, 1, 2, 0): + route(mode) + for graph, _, ws1, ws2 in variants: + ws1.fill_(float("nan")) + ws2.fill_(float("nan")) + graph.replay() + torch.accelerator.synchronize() + torch.testing.assert_close(variants[0][1], variants[1][1], rtol=0, atol=0) + assert torch.isfinite(variants[1][1]).all() + + +@pytest.mark.parametrize(("n", "k"), [(2048, 512), (512, 1024)]) +@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels") +def test_deepgemm_prefix_sum_leaves_unused_capacity_untouched(n, k): + """Both projection shapes skip unused tiles as counts change on replay.""" + from vllm.platforms import current_platform + from vllm.utils.deep_gemm import ( + m_grouped_fp8_gemm_nt_contiguous, + mk_alignment_scope, + ) + + if not ( + current_platform.is_device_capability_family(90) + or current_platform.is_device_capability_family(100) + ): + pytest.skip("Requires SM90 or SM10.x") + + capacity, experts, alignment = 512, 4, 128 + x = torch.randn(capacity, k, device="cuda", dtype=torch.bfloat16) / 10 + aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True) + w, _, ws, _ = make_block_quant_fp8_weights(experts, n // 2, k, BLOCK_SIZE) + ends = torch.tensor([0, 1, 128, 129], device="cuda", dtype=torch.int32) + output = torch.empty(capacity, n, device="cuda", dtype=torch.bfloat16) + + def invoke(): + with mk_alignment_scope(alignment): + m_grouped_fp8_gemm_nt_contiguous( + (aq, scales), (w, ws), output, ends, use_psum_layout=True + ) + + invoke() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + invoke() + for bounds, allocated_end in (([0, 1, 128, 129], 256), ([0] * 4, 0)): + ends.copy_(torch.tensor(bounds, device="cuda", dtype=torch.int32)) + output.fill_(91) + graph.replay() + torch.accelerator.synchronize() + assert torch.all(output[allocated_end:] == 91) + if allocated_end: + assert torch.all(output[[0, 128]] != 91) + + def run_single_case(m, n, k, topk, num_experts, block_size): """Run one (M,N,K) configuration on a single GPU and assert DeepGEMM == Triton baseline within tolerance. diff --git a/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py b/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py index 204954d1fe..b7dcb956c8 100644 --- a/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py +++ b/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py @@ -7,6 +7,7 @@ import torch from vllm.model_executor.layers.quantization.utils.fp8_utils import ( _per_token_group_quant_fp8_colmajor, silu_mul_per_token_group_quant_fp8_colmajor, + silu_mul_quant_fp8_packed_triton, ) from vllm.platforms import current_platform from vllm.triton_utils import triton @@ -108,6 +109,94 @@ def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int): torch.testing.assert_close(output_scales, ref_output_scales) +@pytest.mark.parametrize( + "counts", [[0, 1, 127, 128, 129, 0, 257, 13], [128] * 8, [0] * 8] +) +@pytest.mark.parametrize("group_size", [32, 128]) +@pytest.mark.parametrize("scale_format", ["packed", "float32", "ceil_ue8m0"]) +@pytest.mark.parametrize("hidden_size", [256, 2048]) +@pytest.mark.parametrize("clamp_limit", [None, 10.0]) +@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA graph test") +def test_silu_quant_skips_expert_padding_on_graph_replay( + counts: list[int], + group_size: int, + scale_format: str, + hidden_size: int, + clamp_limit: float | None, +): + """Live rows match dense quantization; gaps stay untouched as counts change.""" + alignment = 128 + capacity = sum(triton.cdiv(n, alignment) * alignment for n in counts) + # Exercise unused capacity as well as the gaps between experts. + capacity = max(capacity, alignment) + if counts != [128] * 8: + capacity += alignment + x = torch.randn((capacity, 2 * hidden_size), device=DEVICE, dtype=torch.bfloat16) + x *= 16 + clean_x = x.clone() + ends = torch.empty(len(counts), device=DEVICE, dtype=torch.int32) + output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=FLOAT8_DTYPE) + + def set_counts(values): + positions = [] + mask = torch.zeros(capacity, device=DEVICE, dtype=torch.bool) + start = 0 + for count in values: + mask[start : start + count] = True + positions.append(start + count) + start += triton.cdiv(count, alignment) * alignment + ends.copy_(torch.tensor(positions, device=DEVICE, dtype=torch.int32)) + return mask + + def quantize(x, output=None, **kwargs): + if scale_format == "packed": + return silu_mul_quant_fp8_packed_triton( + x, + group_size=group_size, + output_q=output, + clamp_limit=clamp_limit, + **kwargs, + ) + return silu_mul_per_token_group_quant_fp8_colmajor( + x, + group_size=group_size, + output=output, + clamp_limit=clamp_limit, + use_ue8m0=scale_format == "ceil_ue8m0", + **kwargs, + ) + + def invoke(): + return quantize( + x, + output=output, + expert_ends=ends, + expert_alignment=alignment, + ) + + set_counts(counts) + invoke() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + actual_q, actual_s = invoke() + + for values in (counts, counts[::-1]): + live = set_counts(values) + x.copy_(clean_x) + x[~live] = float("nan") + output.view(torch.uint8).fill_(91) + graph.replay() + reference_q, reference_s = quantize(x) + torch.testing.assert_close( + actual_q.view(torch.uint8)[live], + reference_q.view(torch.uint8)[live], + rtol=0, + atol=0, + ) + torch.testing.assert_close(actual_s[live], reference_s[live], rtol=0, atol=0) + assert torch.all(actual_q.view(torch.uint8)[~live] == 91) + + @pytest.mark.parametrize("T", [128, 256, 512]) @pytest.mark.parametrize("N", [128 * 2, 256 * 2, 768 * 2, 2048 * 2, 7168 * 2]) @pytest.mark.parametrize("clamp_limit", [7.0, 10.0]) diff --git a/vllm/model_executor/layers/fused_moe/deep_gemm_utils.py b/vllm/model_executor/layers/fused_moe/deep_gemm_utils.py index 33a2296955..624aa9ce86 100644 --- a/vllm/model_executor/layers/fused_moe/deep_gemm_utils.py +++ b/vllm/model_executor/layers/fused_moe/deep_gemm_utils.py @@ -119,6 +119,7 @@ def _fwd_kernel_ep_scatter_1( BLOCK_E: tl.constexpr, BLOCK_EXPERT_NUM: tl.constexpr, ALIGN_M: tl.constexpr, + USE_PSUM_LAYOUT: tl.constexpr, ): cur_expert = tl.program_id(0) @@ -140,20 +141,22 @@ def _fwd_kernel_ep_scatter_1( tl.store(expert_start_loc + cur_expert, cur_expert_start) cur_expert_token_num = tl.load(num_recv_tokens_per_expert + cur_expert) - m_indices_start_ptr = m_indices + cur_expert_start - off_expert = tl.arange(0, BLOCK_E) + if USE_PSUM_LAYOUT: + # Logical ends include preceding experts' alignment gaps. + tl.store(m_indices + cur_expert, cur_expert_start + cur_expert_token_num) + else: + m_indices_start_ptr = m_indices + cur_expert_start + off_expert = tl.arange(0, BLOCK_E) - # any rows in the per-expert aligned region that do not correspond to - # real tokens are left untouched here and should remain initialized to - # -1 so DeepGEMM can skip them - for start_m in tl.range(0, cur_expert_token_num, BLOCK_E): - offs = start_m + off_expert - mask = offs < cur_expert_token_num - tl.store( - m_indices_start_ptr + offs, - cur_expert, - mask=mask, - ) + # Unwritten padding retains the -1 initialization. + for start_m in tl.range(0, cur_expert_token_num, BLOCK_E): + offs = start_m + off_expert + mask = offs < cur_expert_token_num + tl.store( + m_indices_start_ptr + offs, + cur_expert, + mask=mask, + ) @triton.jit @@ -285,6 +288,7 @@ def ep_scatter( align_m: int = 128, block_size: int = 128, pack_ue8m0: bool = False, + use_psum_layout: bool = False, ): # BLOCK_E is the m_indices fill-loop tile (masked), independent of align_m. BLOCK_E = 128 @@ -295,7 +299,10 @@ def ep_scatter( # grid = (triton.cdiv(hidden_size, BLOCK_D), num_experts) grid = num_experts - assert m_indices.shape[0] % align_m == 0 + if use_psum_layout: + assert m_indices.shape == (num_experts,) + else: + assert m_indices.shape[0] % align_m == 0 assert expert_start_loc.shape[0] == num_experts # pack_ue8m0: scatter packs 4 UE8M0 bytes per int32; else copies scales as-is. @@ -311,6 +318,7 @@ def ep_scatter( BLOCK_E=BLOCK_E, BLOCK_EXPERT_NUM=triton.next_power_of_2(num_experts), ALIGN_M=align_m, + USE_PSUM_LAYOUT=use_psum_layout, ) grid = min(recv_topk.shape[0], 1024 * 8) @@ -465,6 +473,7 @@ def deepgemm_moe_permute( expert_tokens_meta: mk.ExpertTokensMetadata | None, aq_out: torch.Tensor | None = None, block_size: int | None = None, + use_psum_layout: bool = False, ): assert aq.ndim == 2 assert topk_ids.dtype.is_signed, "The kernel uses -1 to represent invalid topk_ids" @@ -509,17 +518,15 @@ def deepgemm_moe_permute( else: aq_scale_out = torch.zeros((M_sum, sf_k), device=device, dtype=torch.float32) - # DeepGEMM uses negative values in m_indices (here expert_ids) to mark - # completely invalid / padded blocks that should be skipped. We always - # initialize expert_ids to -1 so any row that is not explicitly written - # by the scatter kernel will be treated as invalid and skipped by - # DeepGEMM's scheduler. - expert_ids = torch.full( - (M_sum,), - fill_value=-1, - device=device, - dtype=torch.int32, - ) + if use_psum_layout: + grouped_layout = torch.empty( + (local_num_experts,), device=device, dtype=torch.int32 + ) + else: + # Negative expert IDs mark padding for the contiguous scheduler. + grouped_layout = torch.full( + (M_sum,), fill_value=-1, device=device, dtype=torch.int32 + ) inv_perm = torch.empty(topk_ids.shape, device=device, dtype=torch.int32) expert_num_tokens = None @@ -541,14 +548,15 @@ def deepgemm_moe_permute( expert_map=expert_map, output_tensor=aq_out, output_tensor_scale=aq_scale_out, - m_indices=expert_ids, + m_indices=grouped_layout, output_index=inv_perm, align_m=align_used, block_size=block_k, pack_ue8m0=pack_ue8m0, + use_psum_layout=use_psum_layout, ) - return aq_out, aq_scale_out, expert_ids, inv_perm, align_used + return aq_out, aq_scale_out, grouped_layout, inv_perm, align_used def deepgemm_unpermute_and_reduce( diff --git a/vllm/model_executor/layers/fused_moe/experts/deep_gemm_moe.py b/vllm/model_executor/layers/fused_moe/experts/deep_gemm_moe.py index 699b029bad..0eef7a8335 100644 --- a/vllm/model_executor/layers/fused_moe/experts/deep_gemm_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/deep_gemm_moe.py @@ -244,7 +244,12 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular): return (workspace1, workspace2, output) def _act_mul_quant( - self, input: torch.Tensor, output: torch.Tensor, activation: MoEActivation + self, + input: torch.Tensor, + output: torch.Tensor, + activation: MoEActivation, + expert_ends: torch.Tensor | None = None, + expert_alignment: int = 0, ) -> tuple[torch.Tensor, torch.Tensor]: assert self.block_shape is not None block_k = self.block_shape[1] @@ -271,6 +276,8 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular): clamp_limit=self.gemm1_clamp_limit, alpha=self.gemm1_alpha, beta=self.gemm1_beta, + expert_ends=expert_ends, + expert_alignment=expert_alignment, ) act_out = torch.empty( (M_sum, activation_out_dim), dtype=input.dtype, device=input.device @@ -294,6 +301,8 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular): group_size=block_k, alpha=self.gemm1_alpha, beta=self.gemm1_beta, + expert_ends=expert_ends, + expert_alignment=expert_alignment, ) # 3. fallback path for non-SiLU activations in non‑UE8M0 cases. @@ -349,7 +358,20 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular): a1q_perm = _resize_cache( workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K) ) - a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute( + # Both GEMMs and quantization must agree on the live expert ranges. + use_psum_layout = ( + not self.mxfp8 + and ( + current_platform.is_device_capability_family(90) + or ( + current_platform.is_device_capability_family(100) + and DeepGemmQuantScaleFMT.from_oracle() + == DeepGemmQuantScaleFMT.UE8M0 + ) + ) + and activation in (MoEActivation.SILU, MoEActivation.SWIGLUOAI_UNINTERLEAVE) + ) + a1q, a1q_scale, grouped_layout, inv_perm, align_used = deepgemm_moe_permute( aq=a1q, aq_scale=a1q_scale, topk_ids=topk_ids, @@ -360,17 +382,21 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular): # MXFP8 uses a 32-element activation-scale group (block_shape[1]); # FP8-block keeps the default (128) alignment. block_size=self.block_shape[1] if self.mxfp8 else None, + use_psum_layout=use_psum_layout, ) assert a1q.size(0) == M_sum # MXFP8 (1x32) drives the fp8_fp4-aliased grouped GEMM with recipe # (1, 32); the FP8 block path keeps the default (128) recipe. - gemm_kwargs = ( + gemm_kwargs: dict = ( {"recipe_a": (1, self.block_shape[1]), "recipe_b": (1, self.block_shape[1])} if self.mxfp8 else {} ) + if use_psum_layout: + gemm_kwargs["use_psum_layout"] = True + # Cap DG's BLOCK_M heuristic at the workspace's per-expert alignment; # otherwise the scheduler can pick the wrong expert id from m_indices # under cudagraph replay. @@ -380,7 +406,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular): (a1q, a1q_scale), (w1, self.w1_scale), mm1_out, - expert_ids, + grouped_layout, **gemm_kwargs, ) @@ -389,7 +415,11 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular): workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim) ) a2q, a2q_scale = self._act_mul_quant( - input=mm1_out.view(-1, N), output=quant_out, activation=activation + input=mm1_out.view(-1, N), + output=quant_out, + activation=activation, + expert_ends=grouped_layout if use_psum_layout else None, + expert_alignment=align_used if use_psum_layout else 0, ) mm2_out = _resize_cache(workspace2, (M_sum, K)) @@ -397,7 +427,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular): (a2q, a2q_scale), (w2, self.w2_scale), mm2_out, - expert_ids, + grouped_layout, **gemm_kwargs, ) diff --git a/vllm/model_executor/layers/quantization/utils/fp8_utils.py b/vllm/model_executor/layers/quantization/utils/fp8_utils.py index c9888dc3cb..fb87c3a614 100644 --- a/vllm/model_executor/layers/quantization/utils/fp8_utils.py +++ b/vllm/model_executor/layers/quantization/utils/fp8_utils.py @@ -142,6 +142,7 @@ def _silu_mul_quant_fp8_packed_kernel( input_ptr, output_q_ptr, output_scale_ptr, + expert_ends_ptr, M, input_stride_m, output_q_stride_m, @@ -158,6 +159,7 @@ def _silu_mul_quant_fp8_packed_kernel( PACKS_PER_CTA: tl.constexpr, BLOCK_M: tl.constexpr, HAS_CLAMP: tl.constexpr, + EXPERT_ALIGNMENT: tl.constexpr, ): GROUPS_PER_PACK: tl.constexpr = 4 hidden_size: tl.constexpr = N // 2 @@ -166,6 +168,13 @@ def _silu_mul_quant_fp8_packed_kernel( row_start = tl.program_id(1).to(tl.int64) * BLOCK_M row_step = tl.num_programs(1).to(tl.int64) * BLOCK_M + if EXPERT_ALIGNMENT: + expert = tl.program_id(2) + previous_end = tl.load(expert_ends_ptr + expert - 1, expert > 0, other=0) + expert_start = tl.cdiv(previous_end, EXPERT_ALIGNMENT) * EXPERT_ALIGNMENT + row_start += expert_start.to(tl.int64) + M = tl.load(expert_ends_ptr + expert) + groups_per_cta: tl.constexpr = PACKS_PER_CTA * GROUPS_PER_PACK elems_per_cta: tl.constexpr = groups_per_cta * GROUP_SIZE col_start = pack_tile * elems_per_cta @@ -254,7 +263,19 @@ def silu_mul_quant_fp8_packed_triton( clamp_limit: float | None = None, alpha: float = 1.0, beta: float = 0.0, + *, + expert_ends: torch.Tensor | None = None, + expert_alignment: int = 0, ) -> tuple[torch.Tensor, torch.Tensor]: + """Fuse activation and FP8 quantization, optionally skipping expert padding. + + ``expert_ends`` uses DeepGEMM's prefix-sum layout: each entry is the + exclusive end of an expert's live rows, including earlier alignment gaps. + An expert starts at the preceding end rounded up to ``expert_alignment``. + Padding outputs are left unwritten and must not be consumed as live rows. + Without ``expert_ends``, the original dense launch configuration is used + and the metadata loads are compiled out. + """ assert input.dim() == 2 assert input.is_contiguous() @@ -293,13 +314,29 @@ def silu_mul_quant_fp8_packed_triton( grid_n = triton.cdiv(packs_per_row, packs_per_cta) grid_m = min(triton.cdiv(M, BM), 4096) - grid = (grid_n, grid_m) + grid: tuple[int, ...] = (grid_n, grid_m) + if expert_ends is not None: + assert expert_ends.ndim == 1 and expert_ends.numel() > 0 + assert expert_ends.dtype == torch.int32 + assert expert_ends.device == input.device and expert_ends.is_contiguous() + assert expert_alignment > 0 + num_experts = expert_ends.numel() + # Bound empty CTAs while retaining enough parallelism for full experts. + max_ctas = 4096 if group_size < 128 else 8192 + grid_m = min( + triton.cdiv(M, BM * num_experts), + max(1, triton.cdiv(max_ctas, grid_n * num_experts)), + ) + grid = (grid_n, grid_m, num_experts) + else: + assert expert_alignment == 0 has_clamp = clamp_limit is not None _silu_mul_quant_fp8_packed_kernel[grid]( input, output_q, output_scale_packed, + expert_ends, M, input.stride(0), output_q.stride(0), @@ -316,6 +353,7 @@ def silu_mul_quant_fp8_packed_triton( PACKS_PER_CTA=packs_per_cta, BLOCK_M=BM, HAS_CLAMP=has_clamp, + EXPERT_ALIGNMENT=expert_alignment, num_warps=num_warps, num_stages=num_stages, ) @@ -328,6 +366,7 @@ def _silu_mul_per_token_group_quant_fp8_colmajor( y_ptr, # [M, N] y_q_ptr, # [M, N // 2] y_s_ptr, # [M, (N // 2) // GROUP_SIZE] + expert_ends_ptr, M, # num tokens N, # intermediate size # Stride @@ -345,8 +384,8 @@ def _silu_mul_per_token_group_quant_fp8_colmajor( GROUP_SIZE: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, + EXPERT_ALIGNMENT: tl.constexpr, ): - # TODO(varun) : Add expert_ids so we may early-exit no-op thread blocks. """Each thread block (BLOCK_N) computes [BLOCK_M, GROUP_SIZE] act-mul outputs. Then the thread block quantizes the [BLOCK_M, GROUP_SIZE] block of values and fills the outputs tensors at the right positions. @@ -357,56 +396,77 @@ def _silu_mul_per_token_group_quant_fp8_colmajor( m_offset = pid_m.to(tl.int64) * BLOCK_M n_offset = pid_n.to(tl.int64) * BLOCK_N - if m_offset >= M: - return + row_step = tl.num_programs(0).to(tl.int64) * BLOCK_M + num_iterations = 1 + if EXPERT_ALIGNMENT: + expert = tl.program_id(2) + previous_end = tl.load(expert_ends_ptr + expert - 1, expert > 0, other=0) + expert_start = tl.cdiv(previous_end, EXPERT_ALIGNMENT) * EXPERT_ALIGNMENT + m_offset += expert_start.to(tl.int64) + M = tl.load(expert_ends_ptr + expert) + num_iterations = tl.cdiv(tl.maximum(M - m_offset, 0), row_step) offs_n = tl.arange(0, BLOCK_N).to(tl.int64) offs_m = tl.arange(0, BLOCK_M).to(tl.int64) - base_y_ptr = y_ptr + m_offset * N + n_offset + for _ in range(num_iterations): + base_y_ptr = y_ptr + m_offset * N + n_offset - act_in_ptrs = base_y_ptr + offs_m[:, None] * N + offs_n[None, :] + act_in_ptrs = base_y_ptr + offs_m[:, None] * N + offs_n[None, :] - act_in = tl.load(act_in_ptrs) - mul_in = tl.load(act_in_ptrs + N_2) + if EXPERT_ALIGNMENT: + live = m_offset + offs_m < M + act_in = tl.load(act_in_ptrs, live[:, None], other=0) + mul_in = tl.load(act_in_ptrs + N_2, live[:, None], other=0) + else: + act_in = tl.load(act_in_ptrs) + mul_in = tl.load(act_in_ptrs + N_2) - # silu & mul — match C++ silu_and_mul: clamp in fp32 then store back to the - # input dtype, run silu in fp32 then narrow, and do the mul at input - # precision so HAS_CLAMP True/False share the same multiplication path. - if HAS_CLAMP: - act_in = tl.minimum(act_in.to(tl.float32), clamp_limit).to( - y_ptr.dtype.element_ty - ) - mul_in = tl.clamp(mul_in.to(tl.float32), -clamp_limit, clamp_limit).to( - y_ptr.dtype.element_ty - ) - # Unified gated activation: silu == swigluoai with alpha=1, beta=0. - # glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu - # Keep glu/up at input precision (narrow before the mul) so the alpha=1, - # beta=0 defaults match the C++ silu_and_mul path bit-for-bit. - act_in = act_in.to(tl.float32) - glu = (act_in / (1.0 + tl.exp(-act_in * alpha))).to(y_ptr.dtype.element_ty) - up = (mul_in.to(tl.float32) + beta).to(y_ptr.dtype.element_ty) - y = (glu * up).to(tl.float32) + # silu & mul — match C++ silu_and_mul: clamp in fp32 then store back to the + # input dtype, run silu in fp32 then narrow, and do the mul at input + # precision so HAS_CLAMP True/False share the same multiplication path. + if HAS_CLAMP: + act_in = tl.minimum(act_in.to(tl.float32), clamp_limit).to( + y_ptr.dtype.element_ty + ) + mul_in = tl.clamp(mul_in.to(tl.float32), -clamp_limit, clamp_limit).to( + y_ptr.dtype.element_ty + ) + # Unified gated activation: silu == swigluoai with alpha=1, beta=0. + # glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu + # Keep glu/up at input precision (narrow before the mul) so the alpha=1, + # beta=0 defaults match the C++ silu_and_mul path bit-for-bit. + act_in = act_in.to(tl.float32) + glu = (act_in / (1.0 + tl.exp(-act_in * alpha))).to(y_ptr.dtype.element_ty) + up = (mul_in.to(tl.float32) + beta).to(y_ptr.dtype.element_ty) + y = (glu * up).to(tl.float32) - # quant - _absmax = tl.maximum(tl.max(tl.abs(y), axis=1), eps) - scale_raw = _absmax * (1.0 / fp8_max) - y_s = tl.math.exp2(tl.ceil(tl.log2(scale_raw))) if use_ue8m0 else scale_raw - y_s = tl.reshape(y_s, (BLOCK_M, 1)) - y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty) + # quant + _absmax = tl.maximum(tl.max(tl.abs(y), axis=1), eps) + scale_raw = _absmax * (1.0 / fp8_max) + y_s = tl.math.exp2(tl.ceil(tl.log2(scale_raw))) if use_ue8m0 else scale_raw + y_s = tl.reshape(y_s, (BLOCK_M, 1)) + y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty) - # store y_q - base_y_q_ptr = y_q_ptr + m_offset * N_2 + n_offset - y_q_ptrs = base_y_q_ptr + offs_m[:, None] * N_2 + offs_n[None, :] - tl.store(y_q_ptrs, y_q) + # store y_q + base_y_q_ptr = y_q_ptr + m_offset * N_2 + n_offset + y_q_ptrs = base_y_q_ptr + offs_m[:, None] * N_2 + offs_n[None, :] + if EXPERT_ALIGNMENT: + tl.store(y_q_ptrs, y_q, live[:, None]) + else: + tl.store(y_q_ptrs, y_q) - # store y_s - group_id = n_offset // GROUP_SIZE - base_y_s_ptr = y_s_ptr + group_id * y_s_col_stride + m_offset - y_s_ptrs = base_y_s_ptr + offs_m - y_s = tl.reshape(y_s, (BLOCK_M,)) - tl.store(y_s_ptrs, y_s) + # store y_s + group_id = n_offset // GROUP_SIZE + base_y_s_ptr = y_s_ptr + group_id * y_s_col_stride + m_offset + y_s_ptrs = base_y_s_ptr + offs_m + y_s = tl.reshape(y_s, (BLOCK_M,)) + if EXPERT_ALIGNMENT: + tl.store(y_s_ptrs, y_s, live) + else: + tl.store(y_s_ptrs, y_s) + + m_offset += row_step def silu_mul_per_token_group_quant_fp8_colmajor( @@ -418,9 +478,15 @@ def silu_mul_per_token_group_quant_fp8_colmajor( group_size: int = 128, alpha: float = 1.0, beta: float = 0.0, + *, + expert_ends: torch.Tensor | None = None, + expert_alignment: int = 0, ): """Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate (silu: alpha=1, beta=0; swigluoai: alpha, beta from config). + + Optional expert endpoints follow DeepGEMM's prefix-sum layout. Padding + values and scales are left unwritten and must not be consumed as live rows. """ GROUP_SIZE = group_size assert input.ndim == 2 @@ -458,13 +524,27 @@ def silu_mul_per_token_group_quant_fp8_colmajor( # Force even division so we can avoid edgecases within the kernel. assert M % BLOCK_M == 0 assert N_2 % BLOCK_N == 0 - grid = (M // BLOCK_M, N_2 // BLOCK_N) + grid: tuple[int, ...] = (M // BLOCK_M, N_2 // BLOCK_N) + if expert_ends is not None: + assert expert_ends.ndim == 1 and expert_ends.numel() > 0 + assert expert_ends.dtype == torch.int32 + assert expert_ends.device == input.device and expert_ends.is_contiguous() + assert expert_alignment > 0 and expert_alignment % BLOCK_M == 0 + num_experts = expert_ends.numel() + grid_m = min( + triton.cdiv(M, BLOCK_M * num_experts), + max(1, triton.cdiv(8192, grid[1] * num_experts)), + ) + grid = (grid_m, grid[1], num_experts) + else: + assert expert_alignment == 0 has_clamp = clamp_limit is not None _silu_mul_per_token_group_quant_fp8_colmajor[grid]( input, output, output_scales, + expert_ends, M, N, output_scales.stride(-1), @@ -479,6 +559,7 @@ def silu_mul_per_token_group_quant_fp8_colmajor( GROUP_SIZE, BLOCK_M, BLOCK_N, + expert_alignment, ) return output, output_scales