[EPLB] Mask padding in EPLB load recording (#38128)

Signed-off-by: ilmarkov <markovilya197@gmail.com>
Signed-off-by: Markov Ilya <markovilya19@gmail.com>
Co-authored-by: Markov Ilya <markovilya19@gmail.com>
This commit is contained in:
Ilya Markov
2026-06-28 19:43:58 -07:00
committed by GitHub
co-authored by Markov Ilya
parent 311ad689ad
commit a2abce646f
17 changed files with 289 additions and 14 deletions
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.fused_moe.router.fused_moe_router import (
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
if current_platform.is_cuda_alike():
@@ -22,11 +23,13 @@ if current_platform.is_cuda_alike():
out_ids_ptr,
out_ptr,
record_enabled_ptr,
num_unpadded_tokens_ptr,
num_logical_experts,
map_slots,
out_size,
numel,
num_active_experts,
HAS_NUM_UNPADDED: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
pid = tl.program_id(0)
@@ -50,6 +53,13 @@ if current_platform.is_cuda_alike():
token_idx = (offs // num_active_experts).to(tl.int64)
hashed = (token_idx * KNUTH_MULTIPLIER) & 0xFFFFFFFF
replica_idx = hashed % replica_count
map_index = safe_expert_id * map_slots + replica_idx
physical_id = tl.load(
logical_to_physical_ptr + map_index,
mask=mask & valid_expert,
other=-1,
)
tl.store(out_ids_ptr + offs, physical_id, mask=mask)
# 2. Record expert load metrics.
@@ -64,16 +74,21 @@ if current_platform.is_cuda_alike():
# If later refactor moved all the MoE kernel calls
# to the modular kernel, we can move this logic there
# to achieve better efficiency.
map_index = safe_expert_id * map_slots + replica_idx
physical_id = tl.load(
logical_to_physical_ptr + map_index,
mask=mask & valid_expert,
other=-1,
)
tl.store(out_ids_ptr + offs, physical_id, mask=mask)
record_enabled = tl.load(record_enabled_ptr) != 0
valid = mask & record_enabled & (physical_id >= 0) & (physical_id < out_size)
# Skip padded tokens when recording.
if HAS_NUM_UNPADDED:
num_unpadded_tokens = tl.load(num_unpadded_tokens_ptr)
is_unpadded = offs < num_unpadded_tokens * num_active_experts
else:
is_unpadded = True
valid = (
mask
& record_enabled
& is_unpadded
& (physical_id >= 0)
& (physical_id < out_size)
)
safe_physical_id = tl.where(physical_id >= 0, physical_id, 0)
tl.atomic_add(out_ptr + safe_physical_id, 1, mask=valid)
@@ -83,6 +98,7 @@ if current_platform.is_cuda_alike():
logical_replica_count: torch.Tensor,
expert_load_view: torch.Tensor,
record_enabled: torch.Tensor,
num_unpadded_tokens: torch.Tensor | None,
) -> torch.Tensor:
topk_ids_in = topk_ids.contiguous().to(dtype=torch.int32)
numel = topk_ids_in.numel()
@@ -99,11 +115,13 @@ if current_platform.is_cuda_alike():
out_flat,
expert_load_view,
record_enabled,
num_unpadded_tokens,
logical_replica_count.shape[0],
logical_to_physical_map.shape[1],
expert_load_view.shape[0],
numel,
num_active_experts,
HAS_NUM_UNPADDED=num_unpadded_tokens is not None,
BLOCK_SIZE=256,
)
return out_flat.reshape(topk_ids.shape)
@@ -114,6 +132,7 @@ if current_platform.is_cuda_alike():
logical_to_physical_map: torch.Tensor,
logical_replica_count: torch.Tensor,
record_enabled: torch.Tensor,
num_unpadded_tokens: torch.Tensor | None = None,
) -> torch.Tensor:
# Fused triton implementation: mapping + optional recording in one kernel.
return _eplb_map_and_record_triton(
@@ -122,6 +141,7 @@ if current_platform.is_cuda_alike():
logical_replica_count=logical_replica_count,
expert_load_view=expert_load_view,
record_enabled=record_enabled,
num_unpadded_tokens=num_unpadded_tokens,
)
else:
@@ -131,6 +151,7 @@ else:
logical_to_physical_map: torch.Tensor,
logical_replica_count: torch.Tensor,
record_enabled: torch.Tensor,
num_unpadded_tokens: torch.Tensor | None = None,
) -> torch.Tensor:
return topk_ids
@@ -177,6 +198,8 @@ class BaseRouter(FusedMoERouter):
raise ValueError("EPLB requires logical_replica_count != None")
if eplb_state.should_record_tensor is None:
raise ValueError("EPLB requires should_record_tensor != None")
if eplb_state.num_unpadded_tokens_tensors is None:
raise ValueError("EPLB requires num_unpadded_tokens_tensors != None")
def _apply_eplb_mapping(self, topk_ids: torch.Tensor) -> torch.Tensor:
"""Apply EPLB mapping to convert logical expert IDs to physical expert IDs."""
@@ -186,12 +209,16 @@ class BaseRouter(FusedMoERouter):
assert eplb_state.logical_to_physical_map is not None
assert eplb_state.logical_replica_count is not None
assert eplb_state.should_record_tensor is not None
assert eplb_state.num_unpadded_tokens_tensors is not None
return eplb_map_to_physical_and_record(
topk_ids=topk_ids,
logical_to_physical_map=eplb_state.logical_to_physical_map,
logical_replica_count=eplb_state.logical_replica_count,
expert_load_view=eplb_state.expert_load_view,
record_enabled=eplb_state.should_record_tensor,
num_unpadded_tokens=eplb_state.num_unpadded_tokens_tensors[
dbo_current_ubatch_id()
],
)
return topk_ids