[Bugfix][MoE] Apply expert_map remapping for CUTLASS FP4 MOE expert parallel (#59042)

Signed-off-by: Markov Ilya <markovilya197@gmail.com>
This commit is contained in:
Ilya Markov
2026-10-12 10:02:56 -04:00
committed by GitHub
parent f022975964
commit b3ee560ae8
6 changed files with 470 additions and 59 deletions
+196 -7
View File
@@ -9,7 +9,7 @@ import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import make_dummy_moe_config
from tests.kernels.moe.utils import make_dummy_moe_config, make_test_quant_config
from vllm import _custom_ops as ops
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import (
@@ -26,8 +26,10 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (
)
from vllm.model_executor.layers.fused_moe.config import (
FUSED_MOE_UNQUANTIZED_CONFIG,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
fp8_w8a8_moe_quant_config,
mxfp4_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsFp4,
@@ -41,7 +43,10 @@ from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
)
from vllm.model_executor.layers.fused_moe.oracle import mxfp4 as mxfp4_oracle
from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
from vllm.model_executor.layers.fused_moe.utils import (
moe_kernel_quantize_input,
remap_topk_to_local,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -183,6 +188,189 @@ def test_cutlass_moe_permutation_maps_padding_to_zero():
torch.testing.assert_close(restored[3], torch.zeros_like(restored[3]))
def test_remap_topk_ids_to_local():
device = "cuda"
expert_map = torch.tensor([-1, 0, -1, 1, -1, 2], dtype=torch.int32, device=device)
topk_ids = torch.tensor([[1, 3], [0, 5], [-1, 4]], dtype=torch.int32, device=device)
expected = torch.tensor(
[[0, 1], [-1, 2], [-1, -1]], dtype=torch.int32, device=device
)
actual = remap_topk_to_local(topk_ids, expert_map, out_dtype=torch.int32)
torch.testing.assert_close(actual, expected)
def test_remap_topk_ids_to_local_requires_contiguous_topk_ids():
expert_map = torch.tensor([0, 1], dtype=torch.int32)
topk_ids = torch.tensor([[0, 1], [1, 0]], dtype=torch.int32).t()
assert not topk_ids.is_contiguous()
with pytest.raises(AssertionError):
remap_topk_to_local(topk_ids, expert_map)
def test_cutlass_fp4_moe_supports_expert_parallel_config():
ep_config = FusedMoEParallelConfig(
tp_size=1,
tp_rank=0,
pcp_size=1,
pcp_rank=0,
dp_size=8,
dp_rank=0,
ep_size=8,
ep_rank=0,
sp_size=1,
use_ep=True,
all2all_backend="allgather_reducescatter",
enable_eplb=False,
)
assert CutlassExpertsMxfp4._supports_parallel_config(ep_config)
assert CutlassExpertsFp4._supports_parallel_config(ep_config)
def _slice_moe_quant_config(
quant_config: FusedMoEQuantConfig, start: int, end: int
) -> FusedMoEQuantConfig:
num_experts = quant_config.w1_scale.shape[0]
sliced_config = copy.deepcopy(quant_config)
for desc_name in ("_a1", "_a2", "_w1", "_w2"):
desc = getattr(sliced_config, desc_name)
for field_name in ("scale", "alpha_or_gscale", "bias", "zp"):
tensor = getattr(desc, field_name)
if (
isinstance(tensor, torch.Tensor)
and tensor.ndim > 0
and tensor.shape[0] == num_experts
):
setattr(desc, field_name, tensor[start:end])
return sliced_config
def _swizzle_mxfp4_moe_scales(
w1_scale: torch.Tensor,
w2_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
e, w1_n, w1_scale_k = w1_scale.shape
w1_k = w1_scale_k * 32
w2_m, w2_scale_n = w2_scale.shape[1], w2_scale.shape[2]
w2_n = w2_scale_n * 32
swizzled_w1 = []
swizzled_w2 = []
for expert in range(e):
swizzled_w1.append(
swizzle_mxfp4_scales(w1_scale[expert], w1_n, w1_k).reshape(w1_n, w1_scale_k)
)
swizzled_w2.append(
swizzle_mxfp4_scales(w2_scale[expert], w2_m, w2_n).reshape(w2_m, w2_scale_n)
)
return torch.stack(swizzled_w1), torch.stack(swizzled_w2)
def _make_cutlass_fp4_moe_weights(
quantization: str,
num_experts: int,
n: int,
k: int,
dtype: torch.dtype,
device: torch.device,
) -> tuple[torch.Tensor, torch.Tensor, FusedMoEQuantConfig]:
if quantization == "nvfp4":
return make_test_quant_config(
num_experts,
n,
k,
in_dtype=dtype,
quant_dtype="nvfp4",
block_shape=None,
per_act_token_quant=False,
make_gate=True,
)
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
mxfp4_quantize,
)
w1, w1_scale = mxfp4_quantize(
torch.randn((num_experts, 2 * n, k), device=device, dtype=dtype) / 8
)
w2, w2_scale = mxfp4_quantize(
torch.randn((num_experts, k, n), device=device, dtype=dtype) / 8
)
w1_scale, w2_scale = _swizzle_mxfp4_moe_scales(w1_scale, w2_scale)
return (
w1.contiguous(),
w2.contiguous(),
mxfp4_moe_quant_config(w1_scale=w1_scale, w2_scale=w2_scale),
)
@pytest.mark.parametrize("quantization", ["nvfp4", "mxfp4"])
@torch.inference_mode()
def test_cutlass_fp4_moe_EP(
quantization: str,
workspace_init,
):
"""Sharded CUTLASS FP4/MXFP4 MoE with expert_map must match full weights."""
experts_cls = CutlassExpertsFp4 if quantization == "nvfp4" else CutlassExpertsMxfp4
if not experts_cls._supports_current_device():
pytest.skip(f"CUTLASS {quantization} MoE is not supported on this GPU")
set_random_seed(7)
m, n, k = 8, 128, 128
num_experts, num_local_experts, topk = 8, 2, 2
dtype = torch.bfloat16
device = torch.device("cuda")
w1, w2, quant_config = _make_cutlass_fp4_moe_weights(
quantization, num_experts, n, k, dtype, device
)
hidden_states = torch.randn((m, k), device=device, dtype=dtype) / 10
score = torch.randn((m, num_experts), device=device, dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(
hidden_states, score, topk, renormalize=False
)
moe_config = make_dummy_moe_config(
num_experts=num_experts,
num_local_experts=num_experts,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=n,
in_dtype=dtype,
max_num_tokens=m,
)
kernel = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
experts_cls(moe_config=moe_config, quant_config=quant_config),
)
apply_kwargs = {
"hidden_states": hidden_states,
"w1": w1,
"w2": w2,
"topk_weights": topk_weights,
"topk_ids": topk_ids,
"global_num_experts": num_experts,
"activation": MoEActivation.SILU,
"expert_map": None,
"apply_router_weight_on_input": False,
}
reference = kernel.apply(**apply_kwargs)
sharded = run_with_expert_maps(
num_experts,
num_local_experts,
quant_config,
experts_cls,
**apply_kwargs,
)
assert reference.abs().max() > 0
torch.testing.assert_close(sharded, reference, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("quantization", ["nvfp4", "mxfp4"])
@torch.inference_mode()
def test_cutlass_fp4_moe_padded_routes_do_not_change_valid_output(quantization: str):
@@ -486,6 +674,7 @@ def run_with_expert_maps(
num_experts: int,
num_local_experts: int,
quant_config: FusedMoEQuantConfig,
experts_cls: type[mk.FusedMoEExperts],
**cutlass_moe_kwargs,
):
def slice_experts():
@@ -513,9 +702,7 @@ def run_with_expert_maps(
for k, t in full_tensors.items():
cutlass_moe_kwargs[k] = t[s:e]
new_quant_config = copy.deepcopy(quant_config)
new_quant_config._w1.scale = quant_config.w1_scale[s:e]
new_quant_config._w2.scale = quant_config.w2_scale[s:e]
new_quant_config = _slice_moe_quant_config(quant_config, s, e)
yield cutlass_moe_kwargs, new_quant_config
@@ -523,13 +710,14 @@ def run_with_expert_maps(
for kwargs, new_quant_config in slice_experts():
w2 = kwargs["w2"]
a = kwargs["hidden_states"]
w1 = kwargs["w1"]
moe_config = make_dummy_moe_config(
max_num_tokens=kwargs.get("hidden_states").shape[0],
experts_per_token=kwargs.get("topk_ids").shape[1],
num_experts=num_experts,
num_local_experts=num_local_experts,
hidden_dim=w2.shape[1],
intermediate_size=w2.shape[2],
intermediate_size=w1.shape[1] // 2,
in_dtype=a.dtype,
)
kernel = mk.FusedMoEKernel(
@@ -539,7 +727,7 @@ def run_with_expert_maps(
allow_new_interface=True,
use_monolithic=False,
),
CutlassExpertsFp8(
experts_cls(
moe_config=moe_config,
quant_config=new_quant_config,
),
@@ -624,6 +812,7 @@ def run_8_bit(
num_experts,
num_local_experts,
quant_config,
CutlassExpertsFp8,
**kwargs,
)
+111
View File
@@ -2114,6 +2114,117 @@ def test_aiter_mxfp4_monolithic_rejects_expert_parallel(
assert "parallel config" in reason
@pytest.mark.parametrize(
"all2all_backend",
[
"deepep_high_throughput",
"flashinfer_nvlink_two_sided",
"flashinfer_nvlink_one_sided",
],
)
def test_make_mxfp4_moe_kernel_rejects_cutlass_unsupported_expert_parallel(
monkeypatch: pytest.MonkeyPatch,
all2all_backend: str,
):
"""Compressed-tensors CUTLASS W4A4 bypasses the oracle; guard at assembly."""
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEParallelConfig,
mxfp4_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsMxfp4,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
make_mxfp4_moe_kernel,
)
monkeypatch.setattr(
CutlassExpertsMxfp4, "_supports_current_device", staticmethod(lambda: True)
)
moe_config = replace(
make_dummy_moe_config(
num_experts=256,
num_local_experts=32,
hidden_dim=4096,
intermediate_size=2048,
),
moe_parallel_config=FusedMoEParallelConfig(
tp_size=1,
tp_rank=0,
pcp_size=1,
pcp_rank=0,
dp_size=8,
dp_rank=0,
ep_size=8,
ep_rank=0,
sp_size=1,
use_ep=True,
all2all_backend=all2all_backend,
enable_eplb=False,
),
)
w1_scale = torch.empty(32, 64, 128, dtype=torch.uint8)
w2_scale = torch.empty(32, 4096, 64, dtype=torch.uint8)
quant_config = mxfp4_moe_quant_config(w1_scale=w1_scale, w2_scale=w2_scale)
with pytest.raises(ValueError, match="parallel config"):
make_mxfp4_moe_kernel(
quant_config,
moe_config,
CutlassExpertsMxfp4,
Mxfp4MoeBackend.MARLIN,
)
def test_make_mxfp4_moe_kernel_validates_cutlass_expert_parallel(
monkeypatch: pytest.MonkeyPatch,
):
"""CUTLASS MXFP4 supports EP with allgather_reducescatter after expert_map."""
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsMxfp4,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
_validate_mxfp4_experts_config,
)
monkeypatch.setattr(
CutlassExpertsMxfp4, "_supports_current_device", staticmethod(lambda: True)
)
moe_config = replace(
make_dummy_moe_config(
num_experts=256,
num_local_experts=32,
hidden_dim=4096,
intermediate_size=2048,
),
moe_parallel_config=FusedMoEParallelConfig(
tp_size=1,
tp_rank=0,
pcp_size=1,
pcp_rank=0,
dp_size=8,
dp_rank=0,
ep_size=8,
ep_rank=0,
sp_size=1,
use_ep=True,
all2all_backend="allgather_reducescatter",
enable_eplb=False,
),
)
_validate_mxfp4_experts_config(
CutlassExpertsMxfp4, moe_config, Mxfp4MoeBackend.MARLIN
)
# Every activation-quantizing OCP MX scheme must map to a `quant_dtype` that
# `moe_kernel_quantize_input` actually dispatches on. Its final `else` returns
# the activation untouched, so a name it does not know (e.g. "mxfp6" instead of
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
)
from vllm.model_executor.layers.fused_moe.utils import (
_resize_cache,
remap_topk_to_local,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
@@ -412,10 +413,9 @@ class CutlassExpertsFp8(CutlassExpertsFp8Base):
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
# CutlassExpertsFp8 does not support expert map, which is
# needed for STANDARD activation format kernels in DP/EP mode.
# Note that the BATCHED activation format does not use
# the expert map for identifying experts.
# EP is supported only with allgather_reducescatter (expert_map remapping).
# DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible
# post-dispatch layouts for Standard-format grouped GEMM.
return not (
moe_parallel_config.use_fi_nvl_two_sided_kernels
or moe_parallel_config.use_deepep_ht_kernels
@@ -491,6 +491,15 @@ FLOAT4_E2M1_MAX = scalar_types.float4_e2m1f.max()
FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
def _prepare_topk_ids_for_cutlass_fp4_moe(
topk_ids: torch.Tensor,
expert_map: torch.Tensor | None,
) -> torch.Tensor:
if expert_map is None:
return topk_ids
return remap_topk_to_local(topk_ids, expert_map, out_dtype=torch.int32)
def run_cutlass_moe_fp4(
output: torch.Tensor,
a: torch.Tensor,
@@ -513,6 +522,7 @@ def run_cutlass_moe_fp4(
e: int,
device: torch.device,
apply_router_weight_on_input: bool = False,
expert_map: torch.Tensor | None = None,
*,
activation_config: ApplyMoEActivationConfig | None = None,
) -> None:
@@ -543,6 +553,8 @@ def run_cutlass_moe_fp4(
assumes that topk < k < n to satisfy - up/down projection expectations.
"""
topk_ids = _prepare_topk_ids_for_cutlass_fp4_moe(topk_ids, expert_map)
is_gated = activation.is_gated
# For gated activations (e.g. SiLU), w1 output is 2*n (gate + up).
# For non-gated activations (e.g. SiLU_NO_MUL), w1 output is n (up only).
@@ -729,9 +741,14 @@ class CutlassExpertsFp4(mk.FusedMoEExpertsModular):
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
# CutlassExpertsFp4 does not support expert map, which is
# needed for STANDARD activation format kernels in EP mode.
return moe_parallel_config.ep_size == 1
# EP is supported only with allgather_reducescatter (expert_map remapping).
# DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible
# post-dispatch layouts for Standard-format grouped GEMM.
return not (
moe_parallel_config.use_fi_nvl_two_sided_kernels
or moe_parallel_config.use_deepep_ht_kernels
or moe_parallel_config.use_fi_nvl_one_sided_kernels
)
@staticmethod
def _supports_batch_invariance() -> bool:
@@ -806,6 +823,7 @@ class CutlassExpertsFp4(mk.FusedMoEExpertsModular):
e=e,
device=hidden_states.device,
apply_router_weight_on_input=apply_router_weight_on_input,
expert_map=expert_map,
activation_config=self.activation_config,
)
@@ -828,10 +846,13 @@ def run_cutlass_moe_mxfp4(
e: int,
device: torch.device,
apply_router_weight_on_input: bool = False,
expert_map: torch.Tensor | None = None,
*,
activation_config: ApplyMoEActivationConfig | None = None,
) -> None:
"""MXFP4 x MXFP4 MoE implementation using CUTLASS grouped GEMM."""
topk_ids = _prepare_topk_ids_for_cutlass_fp4_moe(topk_ids, expert_map)
is_gated = activation.is_gated
w1_n = n * 2 if is_gated else n
@@ -1036,7 +1057,14 @@ class CutlassExpertsMxfp4(mk.FusedMoEExpertsModular):
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return moe_parallel_config.ep_size == 1
# EP is supported only with allgather_reducescatter (expert_map remapping).
# DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible
# post-dispatch layouts for Standard-format grouped GEMM.
return not (
moe_parallel_config.use_fi_nvl_two_sided_kernels
or moe_parallel_config.use_deepep_ht_kernels
or moe_parallel_config.use_fi_nvl_one_sided_kernels
)
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
@@ -1103,6 +1131,7 @@ class CutlassExpertsMxfp4(mk.FusedMoEExpertsModular):
e=e,
device=hidden_states.device,
apply_router_weight_on_input=apply_router_weight_on_input,
expert_map=expert_map,
activation_config=self.activation_config,
)
@@ -21,7 +21,10 @@ from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import (
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.fused_moe.utils import _resize_cache
from vllm.model_executor.layers.fused_moe.utils import (
_resize_cache,
remap_topk_to_local,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
@@ -994,49 +997,6 @@ def masked_moe_sum(
)
@triton.jit
def _remap_topk_to_local_kernel(
topk_ids_ptr, # [n] global expert IDs (-1 = invalid)
expert_map_ptr, # [num_experts] global->local (-1 for non-local)
out_ptr, # [n] int64 local expert IDs (-1 for invalid/non-local)
n_elements,
BLOCK: tl.constexpr,
):
pid = tl.program_id(0)
offs = pid * BLOCK + tl.arange(0, BLOCK)
mask = offs < n_elements
tid = tl.load(topk_ids_ptr + offs, mask=mask, other=-1)
# Gather expert_map[tid] for valid (tid >= 0); clamp the index so invalid
# rows don't read OOB, then select -1 for them. Matches
# torch.where(tid >= 0, expert_map[clamp(tid, 0)], -1) -- preserving -1 (a
# plain expert_map[-1] would wrap to a valid local id and misroute).
valid = tid >= 0
idx = tl.where(valid, tid, 0)
local = tl.load(expert_map_ptr + idx, mask=mask, other=-1)
out = tl.where(valid, local.to(tl.int64), -1)
tl.store(out_ptr + offs, out, mask=mask)
def remap_topk_to_local(
topk_ids: torch.Tensor, expert_map: torch.Tensor
) -> torch.Tensor:
"""Fused global->local expert-id mapping over a topk_ids tensor, preserving -1.
Replaces ``torch.where(topk_ids >= 0, expert_map[topk_ids.clamp(min=0)], -1)``
with one kernel. Returns a NEW int64 tensor -- the caller keeps the original
``topk_ids`` as ``global_topk_ids``, so this must not write in place.
(Distinct from ``deep_gemm_utils.apply_expert_map``, which is a scalar
``@triton.jit`` device helper called from within other kernels.)
"""
out = torch.empty_like(topk_ids, dtype=torch.int64)
n = topk_ids.numel()
BLOCK = 1024
grid = (triton.cdiv(n, BLOCK),)
_remap_topk_to_local_kernel[grid](topk_ids, expert_map, out, n, BLOCK=BLOCK)
return out
class BaseOAITritonExperts(mk.FusedMoEExpertsModular):
@property
def expects_unquantized_inputs(self) -> bool:
@@ -2133,6 +2133,59 @@ def make_mxfp4_moe_quant_config(
)
_MXFP4_SCHEME_CANDIDATES: list[tuple[QuantKey, QuantKey | None]] = [
(kMxfp4Static, kMxfp4Dynamic),
(kMxfp4Static, kMxfp8Dynamic),
(kMxfp4Static, kFp8Dynamic128Sym),
(kMxfp4Static, kFp8StaticTensorSym),
(kMxfp4Static, None),
]
def _mxfp4_quant_keys_for_experts(
experts_cls: type[mk.FusedMoEExperts],
mxfp4_backend: Mxfp4MoeBackend,
) -> tuple[QuantKey | None, QuantKey | None]:
for weight_key, activation_key in _MXFP4_SCHEME_CANDIDATES:
if experts_cls._supports_quant_scheme(weight_key, activation_key):
return weight_key, activation_key
return kMxfp4Static, _backend_activation_key(mxfp4_backend)
def _activation_format_for_config(
moe_config: FusedMoEConfig,
) -> mk.FusedMoEActivationFormat:
return (
mk.FusedMoEActivationFormat.BatchedExperts
if moe_config.moe_parallel_config.use_batched_activation_format
else mk.FusedMoEActivationFormat.Standard
)
def _validate_mxfp4_experts_config(
experts_cls: type[mk.FusedMoEExperts],
moe_config: FusedMoEConfig,
mxfp4_backend: Mxfp4MoeBackend,
) -> None:
"""Fail closed when a caller bypasses the oracle and picks experts directly."""
activation_format = _activation_format_for_config(moe_config)
weight_key, activation_key = _mxfp4_quant_keys_for_experts(
experts_cls, mxfp4_backend
)
supported, reason = experts_cls.is_supported_config(
experts_cls,
moe_config,
weight_key,
activation_key,
activation_format,
)
if not supported:
raise ValueError(
f"{experts_cls.__name__} does not support this MoE deployment"
f"{f' since {reason}' if reason else ''}."
)
def make_mxfp4_moe_kernel(
moe_quant_config: FusedMoEQuantConfig,
moe_config: FusedMoEConfig,
@@ -2141,6 +2194,8 @@ def make_mxfp4_moe_kernel(
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEKernel:
"""Create a FusedMoEKernel for the given MXFP4 backend."""
_validate_mxfp4_experts_config(experts_cls, moe_config, mxfp4_backend)
is_monolithic = issubclass(experts_cls, mk.FusedMoEExpertsMonolithic)
prepare_finalize = maybe_make_prepare_finalize(
@@ -235,6 +235,73 @@ def count_expert_num_tokens(
return expert_num_tokens
@triton.jit
def _remap_topk_to_local_kernel(
topk_ids_ptr, # [n] global expert IDs (-1 = invalid)
expert_map_ptr, # [num_experts] global->local (-1 for non-local)
out_ptr, # [n] local expert IDs (-1 for invalid/non-local)
n_elements,
BLOCK: tl.constexpr,
OUT_INT32: tl.constexpr,
):
pid = tl.program_id(0)
offs = pid * BLOCK + tl.arange(0, BLOCK)
mask = offs < n_elements
tid = tl.load(topk_ids_ptr + offs, mask=mask, other=-1)
# Gather expert_map[tid] for valid (tid >= 0); clamp the index so invalid
# rows don't read OOB, then select -1 for them. Matches
# torch.where(tid >= 0, expert_map[clamp(tid, 0)], -1) -- preserving -1 (a
# plain expert_map[-1] would wrap to a valid local id and misroute).
valid = tid >= 0
idx = tl.where(valid, tid, 0)
local = tl.load(expert_map_ptr + idx, mask=mask, other=-1)
if OUT_INT32:
out = tl.where(valid, local, -1)
else:
out = tl.where(valid, local.to(tl.int64), -1)
tl.store(out_ptr + offs, out, mask=mask)
def remap_topk_to_local(
topk_ids: torch.Tensor,
expert_map: torch.Tensor,
*,
out_dtype: torch.dtype = torch.int64,
) -> torch.Tensor:
"""Map global expert ids to local ids; non-local or invalid routes become -1.
Replaces ``torch.where(topk_ids >= 0, expert_map[topk_ids.clamp(min=0)], -1)``
with one kernel. Returns a NEW tensor -- the caller keeps the original
``topk_ids`` as ``global_topk_ids``, so this must not write in place.
Args:
topk_ids: Global expert ids; -1 marks invalid slots. Must be contiguous;
the Triton kernel indexes by flat ``numel()`` offsets.
expert_map: Global-to-local mapping; -1 marks non-local experts.
out_dtype: Output dtype, ``torch.int32`` or ``torch.int64``.
Returns:
Remapped local expert ids with the same shape as ``topk_ids``.
"""
if out_dtype not in (torch.int32, torch.int64):
raise ValueError(f"out_dtype must be int32 or int64, got {out_dtype}")
assert topk_ids.is_contiguous()
out = torch.empty(topk_ids.shape, dtype=out_dtype, device=topk_ids.device)
n = topk_ids.numel()
BLOCK = 1024
grid = (triton.cdiv(n, BLOCK),)
_remap_topk_to_local_kernel[grid](
topk_ids,
expert_map,
out,
n,
BLOCK=BLOCK,
OUT_INT32=out_dtype == torch.int32,
)
return out
def _resize_cache(x: torch.Tensor, v: tuple[int, ...]) -> torch.Tensor:
"""Shrink the given tensor and apply the given view to it. This is
used to resize the intermediate fused_moe caches.