[ROCm][MoE] Pad the AITER MoE intermediate size at allocation time, and round the expert-group count to a kernel that exists (#55368)
Signed-off-by: Sergei Shliapnikov <sergei.shliapnikov@amd.com>
This commit is contained in:
@@ -30,7 +30,7 @@ import torch.nn.functional as F
|
||||
|
||||
from tests.kernels.utils import _assert_deterministic
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_gfx942, on_gfx950
|
||||
from vllm.platforms.rocm import on_cdna, on_gfx942, on_gfx950
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
@@ -1317,3 +1317,252 @@ def test_aiter_fused_moe_mi3xx_fp8_accuracy():
|
||||
pass_rate=1.0,
|
||||
max_violation_factor=1.5,
|
||||
)
|
||||
|
||||
|
||||
# Weight alignment tests --------------------------------------------------
|
||||
#
|
||||
# AITER's CK 2stages MoE kernel rejects an intermediate size not divisible by
|
||||
# its tile width (64 at or below inter_dim 192, 128 above). Some model + TP
|
||||
# splits land on an unaligned size (e.g. 1792 / TP=8 = 224), so the AITER path
|
||||
# rounds the intermediate dim up in ``maybe_roundup_sizes`` and allocates the
|
||||
# weights at the padded size.
|
||||
|
||||
ALIGNMENT_HIDDEN = 64
|
||||
ALIGNMENT_NUM_EXPERTS = 2
|
||||
|
||||
|
||||
def _make_alignment_moe_config(intermediate: int):
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
|
||||
return make_dummy_moe_config(
|
||||
num_experts=ALIGNMENT_NUM_EXPERTS,
|
||||
hidden_dim=ALIGNMENT_HIDDEN,
|
||||
intermediate_size=intermediate,
|
||||
)
|
||||
|
||||
|
||||
def _make_aiter_method(moe_config):
|
||||
"""Build the unquantized method with the backend pinned to AITER.
|
||||
|
||||
``select_unquantized_moe_backend`` needs a real ROCm + AITER runtime, so
|
||||
stub it out and set the backend directly.
|
||||
"""
|
||||
from unittest.mock import patch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
|
||||
with patch(
|
||||
"vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method"
|
||||
".select_unquantized_moe_backend",
|
||||
return_value=(UnquantizedMoeBackend.AITER, None),
|
||||
):
|
||||
return UnquantizedFusedMoEMethod(moe_config)
|
||||
|
||||
|
||||
def _roundup(method, moe_config, intermediate):
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
|
||||
|
||||
return method.maybe_roundup_sizes(
|
||||
hidden_size=ALIGNMENT_HIDDEN,
|
||||
intermediate_size_per_partition=intermediate,
|
||||
act_dtype=moe_config.in_dtype,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("intermediate", "expected_padded"),
|
||||
[
|
||||
(64, 64), # already aligned, untouched
|
||||
(96, 128),
|
||||
(160, 192),
|
||||
(192, 192), # must stay 192: it is valid and has tuned configs
|
||||
(224, 256), # K2-Horizon-375B at TP=8
|
||||
(256, 256),
|
||||
(448, 512), # K2-Horizon-375B at TP=4
|
||||
(4096, 4096),
|
||||
],
|
||||
)
|
||||
def test_aiter_moe_roundup_pads_intermediate(
|
||||
intermediate,
|
||||
expected_padded,
|
||||
default_vllm_config,
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
aiter_moe_intermediate_alignment,
|
||||
)
|
||||
|
||||
moe_config = _make_alignment_moe_config(intermediate)
|
||||
method = _make_aiter_method(moe_config)
|
||||
|
||||
hidden, padded = _roundup(method, moe_config, intermediate)
|
||||
|
||||
assert padded == expected_padded
|
||||
assert hidden == ALIGNMENT_HIDDEN
|
||||
# Padding must not move a size across the <= 192 threshold, or the
|
||||
# alignment we picked is not the one the kernel dispatches the padded
|
||||
# shape to.
|
||||
assert aiter_moe_intermediate_alignment(padded) == (
|
||||
aiter_moe_intermediate_alignment(intermediate)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend_name", ["TRITON", "FLASHINFER_CUTLASS"])
|
||||
def test_aiter_moe_roundup_is_not_applied_to_other_backends(
|
||||
backend_name,
|
||||
default_vllm_config,
|
||||
):
|
||||
"""Backends with no alignment requirement keep the unaligned size."""
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
)
|
||||
|
||||
intermediate = 224
|
||||
moe_config = _make_alignment_moe_config(intermediate)
|
||||
method = _make_aiter_method(moe_config)
|
||||
method.unquantized_backend = UnquantizedMoeBackend[backend_name]
|
||||
|
||||
_, padded = _roundup(method, moe_config, intermediate)
|
||||
|
||||
assert padded == intermediate
|
||||
|
||||
|
||||
@pytest.mark.parametrize("intermediate", [224, 448])
|
||||
def test_aiter_moe_reload_zeroes_intermediate_padding(
|
||||
intermediate,
|
||||
monkeypatch,
|
||||
default_vllm_config,
|
||||
):
|
||||
"""A reload must re-zero pad lanes left dirty in the weight storage.
|
||||
|
||||
The reload writes back only the logical slices, so the pad lanes hold
|
||||
garbage (NaN here) that the AITER conversion must zero. Two passes also
|
||||
cover reloading into storage that already holds a previous conversion.
|
||||
"""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
aiter_moe_intermediate_alignment,
|
||||
)
|
||||
|
||||
alignment = aiter_moe_intermediate_alignment(intermediate)
|
||||
padded = -(-intermediate // alignment) * alignment
|
||||
assert padded != intermediate
|
||||
|
||||
moe_config = _make_alignment_moe_config(intermediate)
|
||||
assert moe_config.is_act_and_mul
|
||||
method = _make_aiter_method(moe_config)
|
||||
# RoutedExperts records the rounded-up size after maybe_roundup_sizes.
|
||||
moe_config.intermediate_size_per_partition = padded
|
||||
layer = torch.nn.Module()
|
||||
layer.moe_config = moe_config
|
||||
|
||||
method.create_weights(
|
||||
layer=layer,
|
||||
num_experts=ALIGNMENT_NUM_EXPERTS,
|
||||
hidden_size=ALIGNMENT_HIDDEN,
|
||||
intermediate_size_per_partition=padded,
|
||||
params_dtype=torch.float32,
|
||||
)
|
||||
|
||||
# Identity shuffle keeps pad lanes sliceable; both stubs need real ROCm.
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.fused_moe.oracle.unquantized"
|
||||
".rocm_aiter_ops.shuffle_weights",
|
||||
lambda w13, w2: (w13, w2),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
method,
|
||||
"_init_moe_kernel",
|
||||
lambda _layer: setattr(method, "moe_kernel", MagicMock()),
|
||||
)
|
||||
|
||||
up = padded # up projection starts one padded block into the fused rows
|
||||
|
||||
for _ in range(2):
|
||||
gate = torch.randn(ALIGNMENT_NUM_EXPERTS, intermediate, ALIGNMENT_HIDDEN)
|
||||
up_proj = torch.randn(ALIGNMENT_NUM_EXPERTS, intermediate, ALIGNMENT_HIDDEN)
|
||||
down = torch.randn(ALIGNMENT_NUM_EXPERTS, ALIGNMENT_HIDDEN, intermediate)
|
||||
|
||||
# NaN the pads, then write only the logical slices like the loader.
|
||||
layer.w13_weight.data.fill_(float("nan"))
|
||||
layer.w2_weight.data.fill_(float("nan"))
|
||||
layer.w13_weight.data[:, :intermediate].copy_(gate)
|
||||
layer.w13_weight.data[:, up : up + intermediate].copy_(up_proj)
|
||||
layer.w2_weight.data[:, :, :intermediate].copy_(down)
|
||||
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
w13 = layer.w13_weight
|
||||
w2 = layer.w2_weight
|
||||
assert torch.equal(w13[:, :intermediate], gate)
|
||||
assert torch.equal(w13[:, up : up + intermediate], up_proj)
|
||||
assert torch.equal(w2[:, :, :intermediate], down)
|
||||
assert torch.all(w13[:, intermediate:up] == 0)
|
||||
assert torch.all(w13[:, up + intermediate :] == 0)
|
||||
assert torch.all(w2[:, :, intermediate:] == 0)
|
||||
|
||||
|
||||
def _aiter_accepts_intermediate(intermediate: int, num_tokens: int) -> bool:
|
||||
"""Run the real AITER CK MoE kernel and report whether it dispatched."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
ActivationMethod,
|
||||
QuantMethod,
|
||||
)
|
||||
|
||||
case = _make_moe_case(
|
||||
num_tokens=num_tokens,
|
||||
hidden_dim=1024,
|
||||
intermediate_dim=intermediate,
|
||||
num_experts=8,
|
||||
topk=2,
|
||||
seed=0,
|
||||
)
|
||||
try:
|
||||
_run_fused_moe(
|
||||
case["hidden_states"],
|
||||
case["w1"],
|
||||
case["w2"],
|
||||
case["topk_weights"],
|
||||
case["topk_ids"],
|
||||
activation_method=int(ActivationMethod.SILU),
|
||||
quant_method=int(QuantMethod.NO),
|
||||
)
|
||||
except RuntimeError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not on_cdna(),
|
||||
reason="CDNA ROCm only",
|
||||
)
|
||||
@pytest.mark.parametrize("intermediate", [192, 224, 256])
|
||||
# With topk=2 and 8 experts, these token counts select block_m 32 / 64 / 128
|
||||
# respectively (AITER's get_block_size_M, on both 256- and 304-CU parts).
|
||||
@pytest.mark.parametrize("num_tokens", [1024, 2560, 5120])
|
||||
def test_aiter_moe_alignment_rule_holds_across_block_m(intermediate, num_tokens):
|
||||
"""The alignment rule must hold at every reachable ``block_m``.
|
||||
|
||||
Stage 1 checks ``NPerBlock`` and stage 2 ``KPerBlock``; both vary with
|
||||
``block_m``, so a size validated at one tile size says nothing about the
|
||||
others.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
aiter_moe_intermediate_alignment,
|
||||
)
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
alignment = aiter_moe_intermediate_alignment(intermediate)
|
||||
padded = -(-intermediate // alignment) * alignment
|
||||
|
||||
assert _aiter_accepts_intermediate(intermediate, num_tokens) == (
|
||||
intermediate % alignment == 0
|
||||
)
|
||||
assert _aiter_accepts_intermediate(padded, num_tokens)
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the AITER biased_grouped_topk expert-group count.
|
||||
|
||||
The AITER kernel is only instantiated for ``NUM_GRP`` in {1, 2, 4, 8}, so the
|
||||
group count derived from ``num_experts`` has to be rounded to a supported
|
||||
value that still divides ``num_experts``. Grouping is a no-op on this path
|
||||
(``topk_group == num_expert_group``), so any such value routes identically;
|
||||
the constraint is purely about which kernel exists.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
|
||||
AITER_MAX_EXPERTS_PER_GROUP as MAX_EXPERTS_PER_GROUP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
|
||||
AITER_SUPPORTED_NUM_GRP as SUPPORTED_NUM_GRP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
|
||||
_aiter_can_use_biased_grouped_topk,
|
||||
_aiter_get_num_expert_group,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
|
||||
grouped_topk,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_cdna
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_experts", "expected"),
|
||||
[
|
||||
(8, 1), # fits in a single group
|
||||
(32, 1), # exactly at the per-group limit
|
||||
(64, 2),
|
||||
(128, 4),
|
||||
(256, 8),
|
||||
(72, 8), # naive ceil gives 3 -> unsupported, rounds up to 8
|
||||
(96, 8), # naive ceil gives 3 -> unsupported, rounds up to 8
|
||||
(160, 8), # naive ceil gives 5 -> unsupported, rounds up to 8
|
||||
(192, 8), # naive ceil gives 6 -> unsupported, rounds up to 8
|
||||
# No supported NUM_GRP divides these within the group-size limit, so
|
||||
# the naive value is kept and the call site declines the kernel.
|
||||
(33, 3),
|
||||
(129, 43),
|
||||
(257, 257),
|
||||
(320, 10),
|
||||
(384, 12),
|
||||
],
|
||||
)
|
||||
def test_group_count_and_guard(num_experts, expected):
|
||||
g = _aiter_get_num_expert_group(num_experts)
|
||||
assert g == expected
|
||||
# The router asserts both unconditionally; rounding must never break them.
|
||||
assert num_experts % g == 0
|
||||
assert num_experts // g <= MAX_EXPERTS_PER_GROUP
|
||||
|
||||
supported = g in SUPPORTED_NUM_GRP
|
||||
# Ample topk, so only the membership check can decline.
|
||||
assert _aiter_can_use_biased_grouped_topk(num_experts, topk=64) == supported
|
||||
if supported:
|
||||
assert _aiter_can_use_biased_grouped_topk(num_experts, topk=g)
|
||||
assert not _aiter_can_use_biased_grouped_topk(num_experts, topk=g - 1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_cdna(), reason="CDNA ROCm only")
|
||||
@pytest.mark.parametrize(
|
||||
"num_experts",
|
||||
[
|
||||
64, # g == 2, not rounded -- covers the no-op claim itself
|
||||
128, # g == 4, not rounded
|
||||
256, # g == 8, not rounded
|
||||
96, # naive 3 -> rounded to 8
|
||||
192, # naive 6 -> rounded to 8
|
||||
],
|
||||
)
|
||||
def test_rounded_group_count_routes_like_the_reference(num_experts):
|
||||
"""The rounded group count must route identically to ``grouped_topk``.
|
||||
|
||||
Gating is continuous random, so exact score ties -- the one place the two
|
||||
may legitimately differ -- have measure zero.
|
||||
"""
|
||||
torch.manual_seed(num_experts)
|
||||
device = "cuda"
|
||||
topk = 8
|
||||
num_tokens = 83 # not a multiple of any warp/tile size
|
||||
g = _aiter_get_num_expert_group(num_experts)
|
||||
assert _aiter_can_use_biased_grouped_topk(num_experts, topk)
|
||||
|
||||
gating = torch.randn(num_tokens, num_experts, dtype=torch.float32, device=device)
|
||||
bias = torch.randn(num_experts, dtype=torch.float32, device=device)
|
||||
|
||||
ref_weights, ref_ids = grouped_topk(
|
||||
hidden_states=torch.empty(num_tokens, 1, device=device),
|
||||
gating_output=gating,
|
||||
topk=topk,
|
||||
renormalize=True,
|
||||
num_expert_group=g,
|
||||
topk_group=g,
|
||||
scoring_func="sigmoid",
|
||||
e_score_correction_bias=bias,
|
||||
)
|
||||
|
||||
weights = torch.empty(num_tokens, topk, dtype=torch.float32, device=device)
|
||||
ids = torch.empty(num_tokens, topk, dtype=torch.int32, device=device)
|
||||
rocm_aiter_ops.biased_grouped_topk(
|
||||
gating,
|
||||
bias,
|
||||
weights,
|
||||
ids,
|
||||
num_expert_group=g,
|
||||
topk_group=g,
|
||||
need_renorm=True,
|
||||
)
|
||||
|
||||
# Neither side promises an order within a token's top-k, so sort by expert
|
||||
# id and carry the weights through the same permutation -- sorting the two
|
||||
# independently would not catch a weight attached to the wrong expert.
|
||||
order = ids.argsort(dim=-1)
|
||||
ref_order = ref_ids.argsort(dim=-1)
|
||||
torch.testing.assert_close(
|
||||
ids.gather(1, order), ref_ids.to(torch.int32).gather(1, ref_order)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
weights.gather(1, order),
|
||||
ref_weights.gather(1, ref_order),
|
||||
atol=1e-4,
|
||||
rtol=1e-4,
|
||||
)
|
||||
@@ -371,6 +371,16 @@ def select_unquantized_moe_backend(
|
||||
)
|
||||
|
||||
|
||||
def aiter_moe_intermediate_alignment(intermediate: int) -> int:
|
||||
"""Intermediate-size alignment required by AITER's CK 2stages MoE kernel.
|
||||
|
||||
AITER dispatches on ``inter_dim <= 192``: below the threshold both stages
|
||||
use 64-wide tiles, above it at least one stage uses a 128-wide tile, and
|
||||
CK's ``IsSupportedArgument`` rejects a size not divisible by that width.
|
||||
"""
|
||||
return 64 if intermediate <= 192 else 128
|
||||
|
||||
|
||||
def unquantized_round_up_hidden_size_and_intermediate_size(
|
||||
backend: UnquantizedMoeBackend,
|
||||
hidden_size: int,
|
||||
@@ -379,9 +389,30 @@ def unquantized_round_up_hidden_size_and_intermediate_size(
|
||||
"""Round up dimensions before allocation to satisfy the selected kernel."""
|
||||
if backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
|
||||
intermediate_size = round_up(intermediate_size, 128)
|
||||
elif backend == UnquantizedMoeBackend.AITER:
|
||||
intermediate_size = round_up(
|
||||
intermediate_size, aiter_moe_intermediate_alignment(intermediate_size)
|
||||
)
|
||||
return hidden_size, intermediate_size
|
||||
|
||||
|
||||
def _zero_intermediate_padding(
|
||||
moe_config: FusedMoEConfig,
|
||||
w13_weight: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
) -> None:
|
||||
"""Zero intermediate padding; reloads rewrite only the checkpoint slices."""
|
||||
unpadded = moe_config.intermediate_size_per_partition_unpadded
|
||||
assert unpadded is not None
|
||||
padded = moe_config.intermediate_size_per_partition
|
||||
if padded <= unpadded:
|
||||
return
|
||||
w13_weight[:, unpadded:padded].zero_()
|
||||
if moe_config.is_act_and_mul:
|
||||
w13_weight[:, padded + unpadded :].zero_()
|
||||
w2_weight[:, :, unpadded:].zero_()
|
||||
|
||||
|
||||
def convert_to_unquantized_kernel_format(
|
||||
unquantized_backend: UnquantizedMoeBackend,
|
||||
moe_config: FusedMoEConfig,
|
||||
@@ -424,6 +455,7 @@ def convert_to_unquantized_kernel_format(
|
||||
return layout.full_gate_weight, layout.full_down_weight
|
||||
|
||||
if unquantized_backend == UnquantizedMoeBackend.AITER:
|
||||
_zero_intermediate_padding(moe_config, w13_weight, w2_weight)
|
||||
w13_weight, w2_weight = rocm_aiter_ops.shuffle_weights(w13_weight, w2_weight)
|
||||
w13_weight.is_shuffled = True
|
||||
w2_weight.is_shuffled = True
|
||||
@@ -445,15 +477,7 @@ def convert_to_unquantized_kernel_format(
|
||||
)
|
||||
moe_config.intermediate_size_per_partition = padded_intermediate
|
||||
|
||||
# Reloads only overwrite checkpoint slices. An earlier in-place
|
||||
# permutation can leave nonzero values in the raw padding slots.
|
||||
unpadded = moe_config.intermediate_size_per_partition_unpadded
|
||||
assert unpadded is not None
|
||||
if padded_intermediate > unpadded:
|
||||
w13_weight[:, unpadded:padded_intermediate].zero_()
|
||||
if is_act_and_mul:
|
||||
w13_weight[:, padded_intermediate + unpadded :].zero_()
|
||||
w2_weight[:, :, unpadded:].zero_()
|
||||
_zero_intermediate_padding(moe_config, w13_weight, w2_weight)
|
||||
|
||||
_cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
|
||||
convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
|
||||
@@ -19,6 +19,11 @@ from vllm.model_executor.layers.fused_moe.router.dsv4_topk import (
|
||||
dsv4_topk,
|
||||
)
|
||||
|
||||
# AITER instantiates biased_grouped_topk only for these NUM_GRP values, with at
|
||||
# most this many experts per group.
|
||||
AITER_SUPPORTED_NUM_GRP = (1, 2, 4, 8)
|
||||
AITER_MAX_EXPERTS_PER_GROUP = 32
|
||||
|
||||
|
||||
def _get_padding_mask(num_tokens: int) -> torch.Tensor | None:
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
@@ -104,17 +109,40 @@ def vllm_topk_softplus_sqrt(
|
||||
|
||||
@functools.lru_cache(maxsize=8)
|
||||
def _aiter_get_num_expert_group(num_experts: int) -> int:
|
||||
_AITER_MAX_EXPERTS_PER_GROUP = 32
|
||||
g = max(1, -(-num_experts // _AITER_MAX_EXPERTS_PER_GROUP))
|
||||
g = max(1, -(-num_experts // AITER_MAX_EXPERTS_PER_GROUP))
|
||||
while num_experts % g != 0:
|
||||
g += 1
|
||||
assert num_experts % g == 0, f"{num_experts=} not divisible by {g=}"
|
||||
assert num_experts // g <= _AITER_MAX_EXPERTS_PER_GROUP, (
|
||||
f"group size {num_experts // g} exceeds limit {_AITER_MAX_EXPERTS_PER_GROUP}"
|
||||
# Grouping is a no-op here (topk_group == num_expert_group), so any divisor
|
||||
# within the limit routes identically and we are free to pick one AITER has
|
||||
# a kernel for. If none fits, the unsupported value is returned as-is --
|
||||
# callers must gate on _aiter_can_use_biased_grouped_topk.
|
||||
if g not in AITER_SUPPORTED_NUM_GRP:
|
||||
g = next(
|
||||
(
|
||||
c
|
||||
for c in sorted(AITER_SUPPORTED_NUM_GRP, reverse=True)
|
||||
if num_experts % c == 0
|
||||
and num_experts // c <= AITER_MAX_EXPERTS_PER_GROUP
|
||||
),
|
||||
g,
|
||||
)
|
||||
assert num_experts // g <= AITER_MAX_EXPERTS_PER_GROUP, (
|
||||
f"group size {num_experts // g} exceeds limit {AITER_MAX_EXPERTS_PER_GROUP}"
|
||||
)
|
||||
return g
|
||||
|
||||
|
||||
def _aiter_can_use_biased_grouped_topk(num_experts: int, topk: int) -> bool:
|
||||
"""Whether AITER has an instantiated kernel for this routing shape.
|
||||
|
||||
Both checks are needed: an unsupported group count can still be small
|
||||
enough to satisfy ``topk >= g`` (num_experts=33 gives g=3).
|
||||
"""
|
||||
g = _aiter_get_num_expert_group(num_experts)
|
||||
return topk >= g and g in AITER_SUPPORTED_NUM_GRP
|
||||
|
||||
|
||||
def fused_topk_bias(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
@@ -229,7 +257,7 @@ def fused_topk_bias(
|
||||
M = hidden_states.size(0)
|
||||
num_experts = gating_output.shape[-1]
|
||||
num_expert_group = _aiter_get_num_expert_group(num_experts)
|
||||
if topk >= num_expert_group:
|
||||
if _aiter_can_use_biased_grouped_topk(num_experts, topk):
|
||||
topk_weights = torch.empty(
|
||||
M, topk, dtype=torch.float32, device=hidden_states.device
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user