diff --git a/tests/kernels/moe/test_gpt_oss_triton_kernels.py b/tests/kernels/moe/test_gpt_oss_triton_kernels.py index c61004acaa..4afcba706c 100644 --- a/tests/kernels/moe/test_gpt_oss_triton_kernels.py +++ b/tests/kernels/moe/test_gpt_oss_triton_kernels.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from dataclasses import dataclass, fields +from dataclasses import dataclass, fields, is_dataclass import pytest import torch @@ -371,3 +371,59 @@ def test_unit_shuffle(): ) assert_close(ref=out_ref, tri=out) + + +@pytest.mark.parametrize("n_tokens", [1, 33, 512]) +@pytest.mark.parametrize("n_experts,topk", [(32, 4), (128, 4)]) +def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk): + """routing_data_from_sparse_topk must produce routing structures + identical to make_routing_data for the same topk result.""" + from vllm.model_executor.layers.fused_moe.experts import ( + gpt_oss_triton_kernels_moe as gptoss_moe, + ) + + make_routing_data = gptoss_moe.make_routing_data + routing_data_from_sparse_topk = gptoss_moe.routing_data_from_sparse_topk + use_legacy_triton_kernels = gptoss_moe.use_legacy_triton_kernels + + if use_legacy_triton_kernels: + pytest.skip("SparseMatrix path requires triton_kernels v3.6.0+") + + from triton_kernels.topk import topk as topk_fn + + torch.manual_seed(0) + logits = torch.randn(n_tokens, n_experts, dtype=torch.bfloat16, device="cuda") + sparse_topk = topk_fn(logits, topk, apply_softmax=True) + assert not isinstance(sparse_topk, tuple) + + rd_new, gather_new, scatter_new = routing_data_from_sparse_topk( + sparse_topk, n_experts + ) + rd_ref, gather_ref, scatter_ref = make_routing_data( + sparse_topk.indx.to(torch.long), sparse_topk.vals, n_experts + ) + + def assert_equivalent(new, ref, path="RoutingData"): + """Recursively compare tensor-valued leaves of routing structures.""" + if isinstance(ref, torch.Tensor): + torch.testing.assert_close(new, ref, msg=lambda m: f"{path}: {m}") + elif isinstance(ref, (int, float, bool, str)) or ref is None: + assert new == ref, f"{path}: {new!r} != {ref!r}" + elif is_dataclass(ref): + for f in fields(ref): + assert_equivalent( + getattr(new, f.name), getattr(ref, f.name), f"{path}.{f.name}" + ) + elif callable(ref): + assert callable(new), f"{path}: callable vs {type(new)}" + elif hasattr(ref, "__dict__"): + for k, v in vars(ref).items(): + assert_equivalent(getattr(new, k), v, f"{path}.{k}") + else: + assert type(new) is type(ref), f"{path}: {type(new)} vs {type(ref)}" + + assert_equivalent(rd_new, rd_ref) + torch.testing.assert_close(gather_new.src_indx, gather_ref.src_indx) + torch.testing.assert_close(gather_new.dst_indx, gather_ref.dst_indx) + torch.testing.assert_close(scatter_new.src_indx, scatter_ref.src_indx) + torch.testing.assert_close(scatter_new.dst_indx, scatter_ref.dst_indx) diff --git a/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py b/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py index da2628277d..8b87597ec2 100644 --- a/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py @@ -578,13 +578,14 @@ def triton_kernel_moe_forward( if sm_first: logits = torch.softmax(logits, dim=-1) topk_result = topk_fn(logits, topk, apply_softmax=not sm_first) + is_sparse_topk = not isinstance(topk_result, tuple) # topk may return a tuple (vals, indx, bitmatrix) or a # SparseMatrix depending on the triton_kernels version. - if isinstance(topk_result, tuple): - topk_weights, topk_ids_raw, _ = topk_result - else: + if is_sparse_topk: topk_weights = topk_result.vals topk_ids_raw = topk_result.indx + else: + topk_weights, topk_ids_raw, _ = topk_result if expert_map is not None: # topk_ids_raw contains global expert IDs - remap to local. @@ -597,11 +598,23 @@ def triton_kernel_moe_forward( effective_expert_map = None effective_global_num_experts = local_num_experts else: - topk_ids = topk_ids_raw.to(torch.long) - routing_data, gather_idx, scatter_idx = make_routing_data( - topk_ids, topk_weights, gating_output.shape[-1] - ) - effective_expert_map = expert_map + if is_sparse_topk: + # v3.6+ topk returns a SparseMatrix whose construction already + # computed the bitmatrix and its metadata; reuse it instead of + # re-packing the bitmatrix and recomputing the metadata in + # make_routing_data. Only the monolithic path reaches this reuse; + # the modular OAITritonExperts (expert-parallel) path calls + # make_routing_data directly. + routing_data, gather_idx, scatter_idx = routing_data_from_sparse_topk( + topk_result, gating_output.shape[-1] + ) + else: + topk_ids = topk_ids_raw.to(torch.long) + routing_data, gather_idx, scatter_idx = make_routing_data( + topk_ids, topk_weights, gating_output.shape[-1] + ) + + effective_expert_map = None effective_global_num_experts = global_num_experts output = torch.empty_like(hidden_states) @@ -782,18 +795,36 @@ def make_routing_data( ) sparse_logits = SparseMatrix(indx=topk_ids, vals=topk_weights, mask=bitmatrix) - dispatch_indx = sparse_logits.mask_metadata.row_sorted_indx - combine_indx = sparse_logits.mask_metadata.col_sorted_indx + return routing_data_from_sparse_topk(sparse_logits, num_local_experts) + + +def routing_data_from_sparse_topk( + sparse_topk, # SparseMatrix + n_expts: int, +) -> tuple["RoutingData", "GatherIndx", "ScatterIndx"]: + """Build routing structures from the ``SparseMatrix`` returned by + ``triton_kernels.topk``, reusing its precomputed bitmatrix metadata. + + ``make_routing_data`` re-packs a bitmatrix from the topk ids and + constructs a second ``SparseMatrix``, recomputing metadata the topk + call already produced; when the topk ids are used as-is (no expert-map + remapping) that work is redundant. + """ + dispatch_indx = sparse_topk.mask_metadata.row_sorted_indx + combine_indx = sparse_topk.mask_metadata.col_sorted_indx ragged_batch_metadata = make_ragged_tensor_metadata( - sparse_logits.mask_metadata.col_sum, + sparse_topk.mask_metadata.col_sum, dispatch_indx.shape[0], ) - gate_scal = sparse_logits.vals.flatten()[combine_indx] + n_expts_act = sparse_topk.indx.shape[1] + # Unlike make_routing_data, no bfloat16 cast: gate_scal inherits the + # gating dtype, which is bfloat16 for all callers of this path. + gate_scal = sparse_topk.vals.flatten()[combine_indx] routing_data = RoutingData( gate_scal, ragged_batch_metadata.block_sizes, - num_local_experts, - num_topk, + n_expts, + n_expts_act, ragged_batch_metadata, ) gather_indx = GatherIndx(combine_indx, dispatch_indx)