[ROCm][MoE] Pad the AITER MoE intermediate size at allocation time, and round the expert-group count to a kernel that exists (#55368)

Signed-off-by: Sergei Shliapnikov <sergei.shliapnikov@amd.com>
This commit is contained in:
Sergey Shlyapnikov
2026-09-30 23:08:21 +08:00
committed by GitHub
parent 2df122e65e
commit 4e1182a3a6
4 changed files with 452 additions and 15 deletions
@@ -371,6 +371,16 @@ def select_unquantized_moe_backend(
)
def aiter_moe_intermediate_alignment(intermediate: int) -> int:
"""Intermediate-size alignment required by AITER's CK 2stages MoE kernel.
AITER dispatches on ``inter_dim <= 192``: below the threshold both stages
use 64-wide tiles, above it at least one stage uses a 128-wide tile, and
CK's ``IsSupportedArgument`` rejects a size not divisible by that width.
"""
return 64 if intermediate <= 192 else 128
def unquantized_round_up_hidden_size_and_intermediate_size(
backend: UnquantizedMoeBackend,
hidden_size: int,
@@ -379,9 +389,30 @@ def unquantized_round_up_hidden_size_and_intermediate_size(
"""Round up dimensions before allocation to satisfy the selected kernel."""
if backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
intermediate_size = round_up(intermediate_size, 128)
elif backend == UnquantizedMoeBackend.AITER:
intermediate_size = round_up(
intermediate_size, aiter_moe_intermediate_alignment(intermediate_size)
)
return hidden_size, intermediate_size
def _zero_intermediate_padding(
moe_config: FusedMoEConfig,
w13_weight: torch.Tensor,
w2_weight: torch.Tensor,
) -> None:
"""Zero intermediate padding; reloads rewrite only the checkpoint slices."""
unpadded = moe_config.intermediate_size_per_partition_unpadded
assert unpadded is not None
padded = moe_config.intermediate_size_per_partition
if padded <= unpadded:
return
w13_weight[:, unpadded:padded].zero_()
if moe_config.is_act_and_mul:
w13_weight[:, padded + unpadded :].zero_()
w2_weight[:, :, unpadded:].zero_()
def convert_to_unquantized_kernel_format(
unquantized_backend: UnquantizedMoeBackend,
moe_config: FusedMoEConfig,
@@ -424,6 +455,7 @@ def convert_to_unquantized_kernel_format(
return layout.full_gate_weight, layout.full_down_weight
if unquantized_backend == UnquantizedMoeBackend.AITER:
_zero_intermediate_padding(moe_config, w13_weight, w2_weight)
w13_weight, w2_weight = rocm_aiter_ops.shuffle_weights(w13_weight, w2_weight)
w13_weight.is_shuffled = True
w2_weight.is_shuffled = True
@@ -445,15 +477,7 @@ def convert_to_unquantized_kernel_format(
)
moe_config.intermediate_size_per_partition = padded_intermediate
# Reloads only overwrite checkpoint slices. An earlier in-place
# permutation can leave nonzero values in the raw padding slots.
unpadded = moe_config.intermediate_size_per_partition_unpadded
assert unpadded is not None
if padded_intermediate > unpadded:
w13_weight[:, unpadded:padded_intermediate].zero_()
if is_act_and_mul:
w13_weight[:, padded_intermediate + unpadded :].zero_()
w2_weight[:, :, unpadded:].zero_()
_zero_intermediate_padding(moe_config, w13_weight, w2_weight)
_cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
convert_moe_weights_to_flashinfer_trtllm_block_layout(
@@ -19,6 +19,11 @@ from vllm.model_executor.layers.fused_moe.router.dsv4_topk import (
dsv4_topk,
)
# AITER instantiates biased_grouped_topk only for these NUM_GRP values, with at
# most this many experts per group.
AITER_SUPPORTED_NUM_GRP = (1, 2, 4, 8)
AITER_MAX_EXPERTS_PER_GROUP = 32
def _get_padding_mask(num_tokens: int) -> torch.Tensor | None:
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
@@ -104,17 +109,40 @@ def vllm_topk_softplus_sqrt(
@functools.lru_cache(maxsize=8)
def _aiter_get_num_expert_group(num_experts: int) -> int:
_AITER_MAX_EXPERTS_PER_GROUP = 32
g = max(1, -(-num_experts // _AITER_MAX_EXPERTS_PER_GROUP))
g = max(1, -(-num_experts // AITER_MAX_EXPERTS_PER_GROUP))
while num_experts % g != 0:
g += 1
assert num_experts % g == 0, f"{num_experts=} not divisible by {g=}"
assert num_experts // g <= _AITER_MAX_EXPERTS_PER_GROUP, (
f"group size {num_experts // g} exceeds limit {_AITER_MAX_EXPERTS_PER_GROUP}"
# Grouping is a no-op here (topk_group == num_expert_group), so any divisor
# within the limit routes identically and we are free to pick one AITER has
# a kernel for. If none fits, the unsupported value is returned as-is --
# callers must gate on _aiter_can_use_biased_grouped_topk.
if g not in AITER_SUPPORTED_NUM_GRP:
g = next(
(
c
for c in sorted(AITER_SUPPORTED_NUM_GRP, reverse=True)
if num_experts % c == 0
and num_experts // c <= AITER_MAX_EXPERTS_PER_GROUP
),
g,
)
assert num_experts // g <= AITER_MAX_EXPERTS_PER_GROUP, (
f"group size {num_experts // g} exceeds limit {AITER_MAX_EXPERTS_PER_GROUP}"
)
return g
def _aiter_can_use_biased_grouped_topk(num_experts: int, topk: int) -> bool:
"""Whether AITER has an instantiated kernel for this routing shape.
Both checks are needed: an unsupported group count can still be small
enough to satisfy ``topk >= g`` (num_experts=33 gives g=3).
"""
g = _aiter_get_num_expert_group(num_experts)
return topk >= g and g in AITER_SUPPORTED_NUM_GRP
def fused_topk_bias(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
@@ -229,7 +257,7 @@ def fused_topk_bias(
M = hidden_states.size(0)
num_experts = gating_output.shape[-1]
num_expert_group = _aiter_get_num_expert_group(num_experts)
if topk >= num_expert_group:
if _aiter_can_use_biased_grouped_topk(num_experts, topk):
topk_weights = torch.empty(
M, topk, dtype=torch.float32, device=hidden_states.device
)