[ROCm][MoE] Pad the AITER MoE intermediate size at allocation time, and round the expert-group count to a kernel that exists (#55368)
Signed-off-by: Sergei Shliapnikov <sergei.shliapnikov@amd.com>
This commit is contained in:
@@ -371,6 +371,16 @@ def select_unquantized_moe_backend(
|
||||
)
|
||||
|
||||
|
||||
def aiter_moe_intermediate_alignment(intermediate: int) -> int:
|
||||
"""Intermediate-size alignment required by AITER's CK 2stages MoE kernel.
|
||||
|
||||
AITER dispatches on ``inter_dim <= 192``: below the threshold both stages
|
||||
use 64-wide tiles, above it at least one stage uses a 128-wide tile, and
|
||||
CK's ``IsSupportedArgument`` rejects a size not divisible by that width.
|
||||
"""
|
||||
return 64 if intermediate <= 192 else 128
|
||||
|
||||
|
||||
def unquantized_round_up_hidden_size_and_intermediate_size(
|
||||
backend: UnquantizedMoeBackend,
|
||||
hidden_size: int,
|
||||
@@ -379,9 +389,30 @@ def unquantized_round_up_hidden_size_and_intermediate_size(
|
||||
"""Round up dimensions before allocation to satisfy the selected kernel."""
|
||||
if backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
|
||||
intermediate_size = round_up(intermediate_size, 128)
|
||||
elif backend == UnquantizedMoeBackend.AITER:
|
||||
intermediate_size = round_up(
|
||||
intermediate_size, aiter_moe_intermediate_alignment(intermediate_size)
|
||||
)
|
||||
return hidden_size, intermediate_size
|
||||
|
||||
|
||||
def _zero_intermediate_padding(
|
||||
moe_config: FusedMoEConfig,
|
||||
w13_weight: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
) -> None:
|
||||
"""Zero intermediate padding; reloads rewrite only the checkpoint slices."""
|
||||
unpadded = moe_config.intermediate_size_per_partition_unpadded
|
||||
assert unpadded is not None
|
||||
padded = moe_config.intermediate_size_per_partition
|
||||
if padded <= unpadded:
|
||||
return
|
||||
w13_weight[:, unpadded:padded].zero_()
|
||||
if moe_config.is_act_and_mul:
|
||||
w13_weight[:, padded + unpadded :].zero_()
|
||||
w2_weight[:, :, unpadded:].zero_()
|
||||
|
||||
|
||||
def convert_to_unquantized_kernel_format(
|
||||
unquantized_backend: UnquantizedMoeBackend,
|
||||
moe_config: FusedMoEConfig,
|
||||
@@ -424,6 +455,7 @@ def convert_to_unquantized_kernel_format(
|
||||
return layout.full_gate_weight, layout.full_down_weight
|
||||
|
||||
if unquantized_backend == UnquantizedMoeBackend.AITER:
|
||||
_zero_intermediate_padding(moe_config, w13_weight, w2_weight)
|
||||
w13_weight, w2_weight = rocm_aiter_ops.shuffle_weights(w13_weight, w2_weight)
|
||||
w13_weight.is_shuffled = True
|
||||
w2_weight.is_shuffled = True
|
||||
@@ -445,15 +477,7 @@ def convert_to_unquantized_kernel_format(
|
||||
)
|
||||
moe_config.intermediate_size_per_partition = padded_intermediate
|
||||
|
||||
# Reloads only overwrite checkpoint slices. An earlier in-place
|
||||
# permutation can leave nonzero values in the raw padding slots.
|
||||
unpadded = moe_config.intermediate_size_per_partition_unpadded
|
||||
assert unpadded is not None
|
||||
if padded_intermediate > unpadded:
|
||||
w13_weight[:, unpadded:padded_intermediate].zero_()
|
||||
if is_act_and_mul:
|
||||
w13_weight[:, padded_intermediate + unpadded :].zero_()
|
||||
w2_weight[:, :, unpadded:].zero_()
|
||||
_zero_intermediate_padding(moe_config, w13_weight, w2_weight)
|
||||
|
||||
_cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
|
||||
convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
|
||||
@@ -19,6 +19,11 @@ from vllm.model_executor.layers.fused_moe.router.dsv4_topk import (
|
||||
dsv4_topk,
|
||||
)
|
||||
|
||||
# AITER instantiates biased_grouped_topk only for these NUM_GRP values, with at
|
||||
# most this many experts per group.
|
||||
AITER_SUPPORTED_NUM_GRP = (1, 2, 4, 8)
|
||||
AITER_MAX_EXPERTS_PER_GROUP = 32
|
||||
|
||||
|
||||
def _get_padding_mask(num_tokens: int) -> torch.Tensor | None:
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
@@ -104,17 +109,40 @@ def vllm_topk_softplus_sqrt(
|
||||
|
||||
@functools.lru_cache(maxsize=8)
|
||||
def _aiter_get_num_expert_group(num_experts: int) -> int:
|
||||
_AITER_MAX_EXPERTS_PER_GROUP = 32
|
||||
g = max(1, -(-num_experts // _AITER_MAX_EXPERTS_PER_GROUP))
|
||||
g = max(1, -(-num_experts // AITER_MAX_EXPERTS_PER_GROUP))
|
||||
while num_experts % g != 0:
|
||||
g += 1
|
||||
assert num_experts % g == 0, f"{num_experts=} not divisible by {g=}"
|
||||
assert num_experts // g <= _AITER_MAX_EXPERTS_PER_GROUP, (
|
||||
f"group size {num_experts // g} exceeds limit {_AITER_MAX_EXPERTS_PER_GROUP}"
|
||||
# Grouping is a no-op here (topk_group == num_expert_group), so any divisor
|
||||
# within the limit routes identically and we are free to pick one AITER has
|
||||
# a kernel for. If none fits, the unsupported value is returned as-is --
|
||||
# callers must gate on _aiter_can_use_biased_grouped_topk.
|
||||
if g not in AITER_SUPPORTED_NUM_GRP:
|
||||
g = next(
|
||||
(
|
||||
c
|
||||
for c in sorted(AITER_SUPPORTED_NUM_GRP, reverse=True)
|
||||
if num_experts % c == 0
|
||||
and num_experts // c <= AITER_MAX_EXPERTS_PER_GROUP
|
||||
),
|
||||
g,
|
||||
)
|
||||
assert num_experts // g <= AITER_MAX_EXPERTS_PER_GROUP, (
|
||||
f"group size {num_experts // g} exceeds limit {AITER_MAX_EXPERTS_PER_GROUP}"
|
||||
)
|
||||
return g
|
||||
|
||||
|
||||
def _aiter_can_use_biased_grouped_topk(num_experts: int, topk: int) -> bool:
|
||||
"""Whether AITER has an instantiated kernel for this routing shape.
|
||||
|
||||
Both checks are needed: an unsupported group count can still be small
|
||||
enough to satisfy ``topk >= g`` (num_experts=33 gives g=3).
|
||||
"""
|
||||
g = _aiter_get_num_expert_group(num_experts)
|
||||
return topk >= g and g in AITER_SUPPORTED_NUM_GRP
|
||||
|
||||
|
||||
def fused_topk_bias(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
@@ -229,7 +257,7 @@ def fused_topk_bias(
|
||||
M = hidden_states.size(0)
|
||||
num_experts = gating_output.shape[-1]
|
||||
num_expert_group = _aiter_get_num_expert_group(num_experts)
|
||||
if topk >= num_expert_group:
|
||||
if _aiter_can_use_biased_grouped_topk(num_experts, topk):
|
||||
topk_weights = torch.empty(
|
||||
M, topk, dtype=torch.float32, device=hidden_states.device
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user