[Perf] Reuse topk SparseMatrix routing metadata in GPT-OSS MoE forward (#45457)
Signed-off-by: Shenglei Fu <sfu@confluent.io> Signed-off-by: Shenglei Fu <117230642+ShengleiFu@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
7f4793eaa3
commit
129087ddab
@@ -1,6 +1,6 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from dataclasses import dataclass, fields
|
from dataclasses import dataclass, fields, is_dataclass
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
@@ -371,3 +371,59 @@ def test_unit_shuffle():
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert_close(ref=out_ref, tri=out)
|
assert_close(ref=out_ref, tri=out)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("n_tokens", [1, 33, 512])
|
||||||
|
@pytest.mark.parametrize("n_experts,topk", [(32, 4), (128, 4)])
|
||||||
|
def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk):
|
||||||
|
"""routing_data_from_sparse_topk must produce routing structures
|
||||||
|
identical to make_routing_data for the same topk result."""
|
||||||
|
from vllm.model_executor.layers.fused_moe.experts import (
|
||||||
|
gpt_oss_triton_kernels_moe as gptoss_moe,
|
||||||
|
)
|
||||||
|
|
||||||
|
make_routing_data = gptoss_moe.make_routing_data
|
||||||
|
routing_data_from_sparse_topk = gptoss_moe.routing_data_from_sparse_topk
|
||||||
|
use_legacy_triton_kernels = gptoss_moe.use_legacy_triton_kernels
|
||||||
|
|
||||||
|
if use_legacy_triton_kernels:
|
||||||
|
pytest.skip("SparseMatrix path requires triton_kernels v3.6.0+")
|
||||||
|
|
||||||
|
from triton_kernels.topk import topk as topk_fn
|
||||||
|
|
||||||
|
torch.manual_seed(0)
|
||||||
|
logits = torch.randn(n_tokens, n_experts, dtype=torch.bfloat16, device="cuda")
|
||||||
|
sparse_topk = topk_fn(logits, topk, apply_softmax=True)
|
||||||
|
assert not isinstance(sparse_topk, tuple)
|
||||||
|
|
||||||
|
rd_new, gather_new, scatter_new = routing_data_from_sparse_topk(
|
||||||
|
sparse_topk, n_experts
|
||||||
|
)
|
||||||
|
rd_ref, gather_ref, scatter_ref = make_routing_data(
|
||||||
|
sparse_topk.indx.to(torch.long), sparse_topk.vals, n_experts
|
||||||
|
)
|
||||||
|
|
||||||
|
def assert_equivalent(new, ref, path="RoutingData"):
|
||||||
|
"""Recursively compare tensor-valued leaves of routing structures."""
|
||||||
|
if isinstance(ref, torch.Tensor):
|
||||||
|
torch.testing.assert_close(new, ref, msg=lambda m: f"{path}: {m}")
|
||||||
|
elif isinstance(ref, (int, float, bool, str)) or ref is None:
|
||||||
|
assert new == ref, f"{path}: {new!r} != {ref!r}"
|
||||||
|
elif is_dataclass(ref):
|
||||||
|
for f in fields(ref):
|
||||||
|
assert_equivalent(
|
||||||
|
getattr(new, f.name), getattr(ref, f.name), f"{path}.{f.name}"
|
||||||
|
)
|
||||||
|
elif callable(ref):
|
||||||
|
assert callable(new), f"{path}: callable vs {type(new)}"
|
||||||
|
elif hasattr(ref, "__dict__"):
|
||||||
|
for k, v in vars(ref).items():
|
||||||
|
assert_equivalent(getattr(new, k), v, f"{path}.{k}")
|
||||||
|
else:
|
||||||
|
assert type(new) is type(ref), f"{path}: {type(new)} vs {type(ref)}"
|
||||||
|
|
||||||
|
assert_equivalent(rd_new, rd_ref)
|
||||||
|
torch.testing.assert_close(gather_new.src_indx, gather_ref.src_indx)
|
||||||
|
torch.testing.assert_close(gather_new.dst_indx, gather_ref.dst_indx)
|
||||||
|
torch.testing.assert_close(scatter_new.src_indx, scatter_ref.src_indx)
|
||||||
|
torch.testing.assert_close(scatter_new.dst_indx, scatter_ref.dst_indx)
|
||||||
|
|||||||
@@ -578,13 +578,14 @@ def triton_kernel_moe_forward(
|
|||||||
if sm_first:
|
if sm_first:
|
||||||
logits = torch.softmax(logits, dim=-1)
|
logits = torch.softmax(logits, dim=-1)
|
||||||
topk_result = topk_fn(logits, topk, apply_softmax=not sm_first)
|
topk_result = topk_fn(logits, topk, apply_softmax=not sm_first)
|
||||||
|
is_sparse_topk = not isinstance(topk_result, tuple)
|
||||||
# topk may return a tuple (vals, indx, bitmatrix) or a
|
# topk may return a tuple (vals, indx, bitmatrix) or a
|
||||||
# SparseMatrix depending on the triton_kernels version.
|
# SparseMatrix depending on the triton_kernels version.
|
||||||
if isinstance(topk_result, tuple):
|
if is_sparse_topk:
|
||||||
topk_weights, topk_ids_raw, _ = topk_result
|
|
||||||
else:
|
|
||||||
topk_weights = topk_result.vals
|
topk_weights = topk_result.vals
|
||||||
topk_ids_raw = topk_result.indx
|
topk_ids_raw = topk_result.indx
|
||||||
|
else:
|
||||||
|
topk_weights, topk_ids_raw, _ = topk_result
|
||||||
|
|
||||||
if expert_map is not None:
|
if expert_map is not None:
|
||||||
# topk_ids_raw contains global expert IDs - remap to local.
|
# topk_ids_raw contains global expert IDs - remap to local.
|
||||||
@@ -597,11 +598,23 @@ def triton_kernel_moe_forward(
|
|||||||
effective_expert_map = None
|
effective_expert_map = None
|
||||||
effective_global_num_experts = local_num_experts
|
effective_global_num_experts = local_num_experts
|
||||||
else:
|
else:
|
||||||
topk_ids = topk_ids_raw.to(torch.long)
|
if is_sparse_topk:
|
||||||
routing_data, gather_idx, scatter_idx = make_routing_data(
|
# v3.6+ topk returns a SparseMatrix whose construction already
|
||||||
topk_ids, topk_weights, gating_output.shape[-1]
|
# computed the bitmatrix and its metadata; reuse it instead of
|
||||||
)
|
# re-packing the bitmatrix and recomputing the metadata in
|
||||||
effective_expert_map = expert_map
|
# make_routing_data. Only the monolithic path reaches this reuse;
|
||||||
|
# the modular OAITritonExperts (expert-parallel) path calls
|
||||||
|
# make_routing_data directly.
|
||||||
|
routing_data, gather_idx, scatter_idx = routing_data_from_sparse_topk(
|
||||||
|
topk_result, gating_output.shape[-1]
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
topk_ids = topk_ids_raw.to(torch.long)
|
||||||
|
routing_data, gather_idx, scatter_idx = make_routing_data(
|
||||||
|
topk_ids, topk_weights, gating_output.shape[-1]
|
||||||
|
)
|
||||||
|
|
||||||
|
effective_expert_map = None
|
||||||
effective_global_num_experts = global_num_experts
|
effective_global_num_experts = global_num_experts
|
||||||
|
|
||||||
output = torch.empty_like(hidden_states)
|
output = torch.empty_like(hidden_states)
|
||||||
@@ -782,18 +795,36 @@ def make_routing_data(
|
|||||||
)
|
)
|
||||||
|
|
||||||
sparse_logits = SparseMatrix(indx=topk_ids, vals=topk_weights, mask=bitmatrix)
|
sparse_logits = SparseMatrix(indx=topk_ids, vals=topk_weights, mask=bitmatrix)
|
||||||
dispatch_indx = sparse_logits.mask_metadata.row_sorted_indx
|
return routing_data_from_sparse_topk(sparse_logits, num_local_experts)
|
||||||
combine_indx = sparse_logits.mask_metadata.col_sorted_indx
|
|
||||||
|
|
||||||
|
def routing_data_from_sparse_topk(
|
||||||
|
sparse_topk, # SparseMatrix
|
||||||
|
n_expts: int,
|
||||||
|
) -> tuple["RoutingData", "GatherIndx", "ScatterIndx"]:
|
||||||
|
"""Build routing structures from the ``SparseMatrix`` returned by
|
||||||
|
``triton_kernels.topk``, reusing its precomputed bitmatrix metadata.
|
||||||
|
|
||||||
|
``make_routing_data`` re-packs a bitmatrix from the topk ids and
|
||||||
|
constructs a second ``SparseMatrix``, recomputing metadata the topk
|
||||||
|
call already produced; when the topk ids are used as-is (no expert-map
|
||||||
|
remapping) that work is redundant.
|
||||||
|
"""
|
||||||
|
dispatch_indx = sparse_topk.mask_metadata.row_sorted_indx
|
||||||
|
combine_indx = sparse_topk.mask_metadata.col_sorted_indx
|
||||||
ragged_batch_metadata = make_ragged_tensor_metadata(
|
ragged_batch_metadata = make_ragged_tensor_metadata(
|
||||||
sparse_logits.mask_metadata.col_sum,
|
sparse_topk.mask_metadata.col_sum,
|
||||||
dispatch_indx.shape[0],
|
dispatch_indx.shape[0],
|
||||||
)
|
)
|
||||||
gate_scal = sparse_logits.vals.flatten()[combine_indx]
|
n_expts_act = sparse_topk.indx.shape[1]
|
||||||
|
# Unlike make_routing_data, no bfloat16 cast: gate_scal inherits the
|
||||||
|
# gating dtype, which is bfloat16 for all callers of this path.
|
||||||
|
gate_scal = sparse_topk.vals.flatten()[combine_indx]
|
||||||
routing_data = RoutingData(
|
routing_data = RoutingData(
|
||||||
gate_scal,
|
gate_scal,
|
||||||
ragged_batch_metadata.block_sizes,
|
ragged_batch_metadata.block_sizes,
|
||||||
num_local_experts,
|
n_expts,
|
||||||
num_topk,
|
n_expts_act,
|
||||||
ragged_batch_metadata,
|
ragged_batch_metadata,
|
||||||
)
|
)
|
||||||
gather_indx = GatherIndx(combine_indx, dispatch_indx)
|
gather_indx = GatherIndx(combine_indx, dispatch_indx)
|
||||||
|
|||||||
Reference in New Issue
Block a user