[EPLB] Optmize eplb mapping and record in router for prefill (#36261)

Signed-off-by: ilmarkov <markovilya197@gmail.com>
This commit is contained in:
Ilya Markov
2026-03-30 19:48:33 +00:00
committed by GitHub
parent 494636b29d
commit 12701e8af2
6 changed files with 336 additions and 64 deletions
@@ -10,61 +10,49 @@ from vllm.model_executor.layers.fused_moe.router.fused_moe_router import (
FusedMoERouter,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
if current_platform.is_cuda_alike():
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
def eplb_map_to_physical_and_record(
topk_ids: torch.Tensor,
expert_load_view: torch.Tensor,
logical_to_physical_map: torch.Tensor,
logical_replica_count: torch.Tensor,
) -> torch.Tensor:
"""
Map the logical expert ids to physical expert ids
and record the expert load metrics.
@triton.jit
def _eplb_map_and_record_i32_kernel(
topk_ids_ptr,
logical_replica_count_ptr,
logical_to_physical_ptr,
out_ids_ptr,
out_ptr,
record_enabled_ptr,
num_logical_experts,
map_slots,
out_size,
numel,
BLOCK_SIZE: tl.constexpr,
):
pid = tl.program_id(0)
offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offs < numel
This will select a pseudo-random replica for each logical expert.
Only used for EPLB.
Args:
topk_ids: The logical expert ids.
expert_load_view: The expert load view.
logical_to_physical_map: The logical to physical map.
logical_replica_count: The logical replica count.
Returns:
The physical expert ids.
"""
expert_id = tl.load(topk_ids_ptr + offs, mask=mask, other=0).to(tl.int64)
valid_expert = (expert_id >= 0) & (expert_id < num_logical_experts)
safe_expert_id = tl.where(valid_expert, expert_id, 0)
# 1. Convert the logical expert ids to physical expert ids
# Directly select a random replica for each logical expert
# In case `indices_type` is not `torch.long` or `torch.int`,
# e.g. `torch.uint32` as required by dispatch/combine kernels
topk_ids_long = topk_ids.long()
# Use (token position) modulo (replica count)
# to deterministically choose a replica
replica_count = logical_replica_count[topk_ids_long]
# Flatten-position based index, reshaped back to `topk_ids` shape
pos_indices = torch.arange(
topk_ids.numel(), device=topk_ids.device, dtype=torch.long
).reshape_as(topk_ids)
# Compute pseudo-random indices by modulo
replica_indices = (pos_indices % replica_count).unsqueeze(-1)
physical_ids = (
logical_to_physical_map[topk_ids_long]
.gather(-1, replica_indices)
.squeeze(-1)
replica_count = tl.load(
logical_replica_count_ptr + safe_expert_id,
mask=mask & valid_expert,
other=1,
)
topk_ids = physical_ids
# Avoid invalid modulo/div by forcing at least 1.
replica_count = tl.maximum(replica_count, 1)
# Match torch.compile path: use flattened token position.
replica_idx = offs % replica_count
# 2. Record expert load metrics.
# TODO(bowen): When using `FusedMoEModularKernel`, this
# can be done in a more unified way, since
# `FusedMoEPrepareAndFinalizeModular` will return the expert
# `FusedMoEPrepareAndFinalize` will return the expert
# token count, in some cases directly from the kernel.
# However, now there are many code paths not using
# the modular kernel, e.g. calling `fused_experts`,
@@ -73,17 +61,63 @@ if current_platform.is_cuda_alike():
# If later refactor moved all the MoE kernel calls
# to the modular kernel, we can move this logic there
# to achieve better efficiency.
map_index = safe_expert_id * map_slots + replica_idx
physical_id = tl.load(
logical_to_physical_ptr + map_index,
mask=mask & valid_expert,
other=-1,
)
tl.store(out_ids_ptr + offs, physical_id, mask=mask)
# `expert_load_view`: (num_physical_experts,)
record_enabled = tl.load(record_enabled_ptr) != 0
valid = mask & record_enabled & (physical_id >= 0) & (physical_id < out_size)
safe_physical_id = tl.where(physical_id >= 0, physical_id, 0)
tl.atomic_add(out_ptr + safe_physical_id, 1, mask=valid)
# `torch.bincount` is not compilable, so use `scatter_add_` instead.
topk_ids_flatten = topk_ids.flatten()
expert_load_view.scatter_add_(
dim=0,
index=topk_ids_flatten.long(),
src=torch.ones_like(topk_ids_flatten).to(expert_load_view),
def _eplb_map_and_record_triton(
topk_ids: torch.Tensor,
logical_to_physical_map: torch.Tensor,
logical_replica_count: torch.Tensor,
expert_load_view: torch.Tensor,
record_enabled: torch.Tensor,
) -> torch.Tensor:
topk_ids_in = topk_ids.contiguous().to(dtype=torch.int32)
numel = topk_ids_in.numel()
if numel == 0:
return topk_ids
out_flat = torch.empty((numel,), device=topk_ids.device, dtype=topk_ids.dtype)
grid = lambda meta: (triton.cdiv(numel, meta["BLOCK_SIZE"]),)
assert expert_load_view.is_contiguous()
_eplb_map_and_record_i32_kernel[grid](
topk_ids_in,
logical_replica_count.contiguous(),
logical_to_physical_map.contiguous(),
out_flat,
expert_load_view,
record_enabled,
logical_replica_count.shape[0],
logical_to_physical_map.shape[1],
expert_load_view.shape[0],
numel,
BLOCK_SIZE=256,
)
return out_flat.reshape(topk_ids.shape)
def eplb_map_to_physical_and_record(
topk_ids: torch.Tensor,
expert_load_view: torch.Tensor,
logical_to_physical_map: torch.Tensor,
logical_replica_count: torch.Tensor,
record_enabled: torch.Tensor,
) -> torch.Tensor:
# Fused triton implementation: mapping + optional recording in one kernel.
return _eplb_map_and_record_triton(
topk_ids=topk_ids,
logical_to_physical_map=logical_to_physical_map,
logical_replica_count=logical_replica_count,
expert_load_view=expert_load_view,
record_enabled=record_enabled,
)
return topk_ids
else:
def eplb_map_to_physical_and_record(
@@ -91,8 +125,8 @@ else:
expert_load_view: torch.Tensor,
logical_to_physical_map: torch.Tensor,
logical_replica_count: torch.Tensor,
record_enabled: torch.Tensor,
) -> torch.Tensor:
# CPU fallback: no EPLB so just return as is
return topk_ids
@@ -146,6 +180,10 @@ class BaseRouter(FusedMoERouter):
raise ValueError(
"enable_eplb=True requires logical_replica_count != None"
)
if self.eplb_state.should_record_tensor is None:
raise ValueError(
"enable_eplb=True requires should_record_tensor != None"
)
def _get_indices_type(self) -> torch.dtype | None:
"""Get the desired indices dtype from the getter function."""
@@ -159,11 +197,13 @@ class BaseRouter(FusedMoERouter):
assert self.eplb_state.expert_load_view is not None
assert self.eplb_state.logical_to_physical_map is not None
assert self.eplb_state.logical_replica_count is not None
assert self.eplb_state.should_record_tensor is not None
return eplb_map_to_physical_and_record(
topk_ids=topk_ids,
expert_load_view=self.eplb_state.expert_load_view,
logical_to_physical_map=self.eplb_state.logical_to_physical_map,
logical_replica_count=self.eplb_state.logical_replica_count,
expert_load_view=self.eplb_state.expert_load_view,
record_enabled=self.eplb_state.should_record_tensor,
)
return topk_ids