diff --git a/tests/kernels/moe/test_gpt_oss_triton_kernels.py b/tests/kernels/moe/test_gpt_oss_triton_kernels.py index 406e525bdf..3962ab9134 100644 --- a/tests/kernels/moe/test_gpt_oss_triton_kernels.py +++ b/tests/kernels/moe/test_gpt_oss_triton_kernels.py @@ -431,11 +431,6 @@ def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk): if gptoss_moe.triton_kernels_version == "3.5.1": pytest.skip("SparseMatrix path requires triton_kernels v3.6.0+") - if gptoss_moe.triton_kernels_version == "3.8": - pytest.skip( - "3.8 rebuilds routing from raw ids, so routing_data_from_sparse_topk " - "just forwards to make_routing_data and the comparison is vacuous" - ) from triton_kernels.topk import topk as topk_fn @@ -467,11 +462,52 @@ def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk): elif hasattr(ref, "__dict__"): for k, v in vars(ref).items(): assert_equivalent(getattr(new, k), v, f"{path}.{k}") + elif hasattr(type(ref), "__slots__"): + for k in type(ref).__slots__: + assert_equivalent(getattr(new, k), getattr(ref, k), f"{path}.{k}") else: assert type(new) is type(ref), f"{path}: {type(new)} vs {type(ref)}" assert_equivalent(rd_new, rd_ref) + if isinstance(gather_ref, torch.Tensor): + torch.testing.assert_close(gather_new, gather_ref) + torch.testing.assert_close(scatter_new, scatter_ref) + return torch.testing.assert_close(gather_new.src_indx, gather_ref.src_indx) torch.testing.assert_close(gather_new.dst_indx, gather_ref.dst_indx) torch.testing.assert_close(scatter_new.src_indx, scatter_ref.src_indx) torch.testing.assert_close(scatter_new.dst_indx, scatter_ref.dst_indx) + + +@pytest.mark.parametrize("n_tokens", [1, 33, 4099]) +@pytest.mark.parametrize("n_experts,topk", [(128, 4), (256, 6)]) +def test_make_routing_data_tk38_matches_reference(n_tokens, n_experts, topk): + from vllm.model_executor.layers.fused_moe.experts import ( + gpt_oss_triton_kernels_moe as gptoss_moe, + ) + + if gptoss_moe.triton_kernels_version != "3.8": + pytest.skip("requires triton_kernels 3.8") + + torch.manual_seed(0) + topk_ids = torch.stack( + [torch.randperm(n_experts, device="cuda")[:topk] for _ in range(n_tokens)] + ).to(torch.int32) + topk_ids[torch.rand(topk_ids.shape, device="cuda") < 0.5] = -1 + topk_weights = torch.rand(topk_ids.shape, device="cuda") + + routing, gather_tok, scatter_tok = gptoss_moe.make_routing_data( + topk_ids, topk_weights, n_experts + ) + + flat = topk_ids.reshape(-1).to(torch.int64) + order = torch.argsort(flat.clamp(min=0), stable=True) + counts = torch.bincount(flat.clamp(min=0), minlength=n_experts).to(torch.int32) + weights = topk_weights.reshape(-1) * (flat >= 0).to(torch.float32) + + assert torch.equal(routing.ragged.slice_sizes, counts) + assert torch.equal(gather_tok, (order // topk).to(torch.int32)) + assert torch.equal(scatter_tok, gather_tok) + assert torch.equal(routing.gate_scal, weights[order]) + assert routing.n_expts_act == topk + assert routing.n_valid == flat.numel() diff --git a/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py b/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py index 3e70c15893..bdc5384685 100644 --- a/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py @@ -847,6 +847,20 @@ def triton_kernel_fused_experts( return output_tensor +def _tk38_routing_data( + col_sum: torch.Tensor, + order: torch.Tensor, + flat_w: torch.Tensor, + topk: int, +) -> tuple["RoutingData", torch.Tensor, torch.Tensor]: + """Build 3.8 routing from per-expert counts and the stable expert order.""" + nrows = order.shape[0] + ragged = make_ragged_tensor_metadata(col_sum, nrows) + gather_tok = (order // topk).to(torch.int32) + routing = RoutingData(ragged, flat_w[order], gather_tok, topk, nrows) + return routing, gather_tok, gather_tok + + def make_routing_data( topk_ids: torch.Tensor, topk_weights: torch.Tensor, @@ -854,21 +868,18 @@ def make_routing_data( ) -> tuple["RoutingData", torch.Tensor, torch.Tensor]: if triton_kernels_version == "3.8": _, topk = topk_ids.shape - flat_e = topk_ids.reshape(-1) - valid = flat_e >= 0 - flat_e = flat_e.to(torch.int64).clamp(min=0) - flat_w = topk_weights.reshape(-1).to(torch.float32) * valid.to(torch.float32) - nrows = flat_e.shape[0] - order = torch.argsort(flat_e, stable=True) - col_sum = torch.zeros( - num_local_experts, dtype=torch.int32, device=flat_e.device + flat_ids = topk_ids.reshape(-1) + # -1 (non-local) slots go to expert 0 with zero weight. + keys = flat_ids.clamp(min=0) + sorted_keys, order = torch.sort(keys, stable=True) + experts = torch.arange( + num_local_experts + 1, dtype=keys.dtype, device=keys.device ) - col_sum.scatter_add_(0, flat_e, torch.ones_like(flat_e, dtype=torch.int32)) - ragged = make_ragged_tensor_metadata(col_sum, nrows) - gather_tok = (order // topk).to(torch.int32) - gammas = flat_w[order] - routing = RoutingData(ragged, gammas, gather_tok, topk, nrows) - return routing, gather_tok, gather_tok + edges = torch.searchsorted(sorted_keys, experts, out_int32=True) + col_sum = edges[1:] - edges[:-1] + valid = (flat_ids >= 0).to(torch.float32) + flat_w = topk_weights.reshape(-1).to(torch.float32) * valid + return _tk38_routing_data(col_sum, order, flat_w, topk) topk_ids = topk_ids.to(torch.int16) topk_weights = topk_weights.to(torch.bfloat16) @@ -936,8 +947,14 @@ def routing_data_from_sparse_topk( remapping) that work is redundant. """ if triton_kernels_version == "3.8": - # 3.8 routing is rebuilt from raw ids; no SparseMatrix reuse. - return make_routing_data(sparse_topk.indx, sparse_topk.vals, n_expts) + # topk ids have no -1 slots, so the topk metadata matches make_routing_data. + mask_metadata = sparse_topk.mask_metadata + return _tk38_routing_data( + mask_metadata.col_sum, + mask_metadata.col_sorted_indx, + sparse_topk.vals.reshape(-1).to(torch.float32), + sparse_topk.indx.shape[1], + ) dispatch_indx = sparse_topk.mask_metadata.row_sorted_indx combine_indx = sparse_topk.mask_metadata.col_sorted_indx