[Bugfix][MoE] Apply expert_map remapping for CUTLASS FP4 MOE expert parallel (#59042)

Signed-off-by: Markov Ilya <markovilya197@gmail.com>
This commit is contained in:
Ilya Markov
2026-10-12 10:02:56 -04:00
committed by GitHub
parent f022975964
commit b3ee560ae8
6 changed files with 470 additions and 59 deletions
@@ -235,6 +235,73 @@ def count_expert_num_tokens(
return expert_num_tokens
@triton.jit
def _remap_topk_to_local_kernel(
topk_ids_ptr, # [n] global expert IDs (-1 = invalid)
expert_map_ptr, # [num_experts] global->local (-1 for non-local)
out_ptr, # [n] local expert IDs (-1 for invalid/non-local)
n_elements,
BLOCK: tl.constexpr,
OUT_INT32: tl.constexpr,
):
pid = tl.program_id(0)
offs = pid * BLOCK + tl.arange(0, BLOCK)
mask = offs < n_elements
tid = tl.load(topk_ids_ptr + offs, mask=mask, other=-1)
# Gather expert_map[tid] for valid (tid >= 0); clamp the index so invalid
# rows don't read OOB, then select -1 for them. Matches
# torch.where(tid >= 0, expert_map[clamp(tid, 0)], -1) -- preserving -1 (a
# plain expert_map[-1] would wrap to a valid local id and misroute).
valid = tid >= 0
idx = tl.where(valid, tid, 0)
local = tl.load(expert_map_ptr + idx, mask=mask, other=-1)
if OUT_INT32:
out = tl.where(valid, local, -1)
else:
out = tl.where(valid, local.to(tl.int64), -1)
tl.store(out_ptr + offs, out, mask=mask)
def remap_topk_to_local(
topk_ids: torch.Tensor,
expert_map: torch.Tensor,
*,
out_dtype: torch.dtype = torch.int64,
) -> torch.Tensor:
"""Map global expert ids to local ids; non-local or invalid routes become -1.
Replaces ``torch.where(topk_ids >= 0, expert_map[topk_ids.clamp(min=0)], -1)``
with one kernel. Returns a NEW tensor -- the caller keeps the original
``topk_ids`` as ``global_topk_ids``, so this must not write in place.
Args:
topk_ids: Global expert ids; -1 marks invalid slots. Must be contiguous;
the Triton kernel indexes by flat ``numel()`` offsets.
expert_map: Global-to-local mapping; -1 marks non-local experts.
out_dtype: Output dtype, ``torch.int32`` or ``torch.int64``.
Returns:
Remapped local expert ids with the same shape as ``topk_ids``.
"""
if out_dtype not in (torch.int32, torch.int64):
raise ValueError(f"out_dtype must be int32 or int64, got {out_dtype}")
assert topk_ids.is_contiguous()
out = torch.empty(topk_ids.shape, dtype=out_dtype, device=topk_ids.device)
n = topk_ids.numel()
BLOCK = 1024
grid = (triton.cdiv(n, BLOCK),)
_remap_topk_to_local_kernel[grid](
topk_ids,
expert_map,
out,
n,
BLOCK=BLOCK,
OUT_INT32=out_dtype == torch.int32,
)
return out
def _resize_cache(x: torch.Tensor, v: tuple[int, ...]) -> torch.Tensor:
"""Shrink the given tensor and apply the given view to it. This is
used to resize the intermediate fused_moe caches.