[ROCm][CI][AiterExperts] Add test coverage for hidden/intermediate padding correctness (#59333)

Signed-off-by: Divakar Verma <divakar.verma@amd.com>
This commit is contained in:
Divakar Verma
2026-10-03 01:53:37 +00:00
committed by GitHub
parent 44198f577f
commit faa9860dbd
2 changed files with 311 additions and 1 deletions
@@ -88,6 +88,15 @@ class Config:
torch_trace_dir_path: str | None = None
# Force AiterExperts's hidden_pad/intermediate_pad computation
# (`experts/rocm_aiter_moe.py`) to diverge from the padded K/N sizes above.
# None (default) preserves today's behavior: FusedMoEConfig defaults both
# to the (unpadded) K/intermediate_size_per_partition, so hidden_pad and
# intermediate_pad come out to 0.
# See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").
hidden_dim_unpadded: int | None = None
intermediate_size_per_partition_unpadded: int | None = None
def __post_init__(self):
if self.quant_config is None:
self.quant_config = TestMoEQuantConfig(None, False, False, None)
@@ -664,6 +673,10 @@ def make_modular_kernel(
activation=config.activation,
device=vllm_config.device_config.device,
routing_method=RoutingMethodType.DeepSeekV3,
hidden_dim_unpadded=config.hidden_dim_unpadded,
intermediate_size_per_partition_unpadded=(
config.intermediate_size_per_partition_unpadded
),
)
prepare_finalize = maybe_make_prepare_finalize(
@@ -42,7 +42,7 @@ from .modular_kernel_tools.parallel_utils import (
ProcessGroupInfo,
parallel_launch_with_config,
)
from .utils import check_accuracy
from .utils import check_accuracy, make_test_weights
has_any_multi_gpu_package = (
has_deep_ep() or has_deep_gemm() or has_flashinfer_cutlass_fused_moe()
@@ -691,6 +691,303 @@ def test_aiter_moe_dispatch_policy_forwarded_through_apply(dispatch_policy: int)
)
# --- 4b: hidden_dim_unpadded/intermediate_size_per_partition_unpadded matrix -
# K/N intentionally unaligned to AITER's 64/128 granularity (unlike the
# shared mk_objects.py defaults) so hidden_pad/intermediate_pad are forced.
_PADDING_E = 8
_PADDING_M = 16
_PADDING_TOPK = 2
_PADDING_K_UNPADDED = 896 # 7 * 128
_PADDING_K_PADDED = 1024 # +128
_PADDING_N_UNPADDED = 384 # 3 * 128
_PADDING_N_PADDED = 512 # +128
# hidden_pad = (K_padded - hidden_dim_unpadded) // 128 * 128 (rocm_aiter_moe.py)
_PADDING_HIDDEN_PAD_EXPECTED = 128
# intermediate_pad = (N_padded - intermediate_size_per_partition_unpadded)
# // 64 * 64 * (2 if tp_size == 1 else 1) (rocm_aiter_moe.py); tp_size==1 here.
_PADDING_INTERMEDIATE_PAD_EXPECTED = 256
# {no padding, hidden-only, intermediate-only, both} -> (pad_hidden, pad_intermediate)
_PADDING_MODES: dict[str, tuple[bool, bool]] = {
"none": (False, False),
"hidden": (True, False),
"intermediate": (False, True),
"both": (True, True),
}
# unquantized + the 4 fp8 configs AiterExperts supports (excludes MXFP4,
# stubbed below pending MI350/gfx950, and MK_QUANT_CONFIGS[1], unsupported).
_PADDING_QUANT_CONFIGS = [
MK_QUANT_CONFIGS[0], # unquantized
MK_QUANT_CONFIGS[2], # fp8 channel weights / per-token activations
MK_QUANT_CONFIGS[3], # fp8 per-tensor weights / per-tensor activations
MK_QUANT_CONFIGS[4], # fp8 per-tensor weights / per-token activations
MK_QUANT_CONFIGS[5], # fp8 128x128-block weights / 128-block activations
]
_PADDING_QUANT_IDS = [
"unquantized",
"fp8_channel_token",
"fp8_tensor_tensor",
"fp8_tensor_token",
"fp8_block_token",
]
def _slice_gate_up_rows(t: torch.Tensor, real_per_half: int) -> torch.Tensor:
"""Slice a (E, 2*padded_per_half, ...) tensor's dim=1 down to the real
gate/up halves of `real_per_half` size each."""
padded_per_half = t.shape[1] // 2
return torch.cat(
[
t[:, :real_per_half],
t[:, padded_per_half : padded_per_half + real_per_half],
],
dim=1,
)
def _slice_unpadded_weights(
weights: WeightTensors,
quant_config: TestMoEQuantConfig | None,
k_unpadded: int,
n_unpadded: int,
) -> WeightTensors:
"""Derive the unpadded sub-block of a padded WeightTensors: same values,
restricted to the region hidden_pad/intermediate_pad keeps.
Scale-shape choice (per_out_ch) must match _make_padding_matrix_weights()."""
block_shape = quant_config.block_shape if quant_config is not None else None
per_out_ch = quant_config is not None and quant_config.per_out_ch_quant
w1 = _slice_gate_up_rows(weights.w1[:, :, :k_unpadded], n_unpadded)
w2 = weights.w2[:, :k_unpadded, :n_unpadded]
if weights.w1_scale is None:
w1_scale = w2_scale = None
elif weights.w2_scale is None:
raise AssertionError("w1_scale and w2_scale must both be set or both None")
elif block_shape is not None:
block_n, block_k = block_shape
assert n_unpadded % block_n == 0 and k_unpadded % block_k == 0
n_scale, k_scale = n_unpadded // block_n, k_unpadded // block_k
w1_scale = _slice_gate_up_rows(weights.w1_scale, n_scale)[..., :k_scale]
w2_scale = weights.w2_scale[:, :k_scale, :n_scale]
elif per_out_ch:
w1_scale = _slice_gate_up_rows(weights.w1_scale, n_unpadded)
w2_scale = weights.w2_scale[:, :k_unpadded, :]
else:
# Per-tensor scale: shape (E, 1, 1), independent of K/N -- reuse as-is.
w1_scale = weights.w1_scale
w2_scale = weights.w2_scale
return WeightTensors(w1=w1, w2=w2, w1_scale=w1_scale, w2_scale=w2_scale)
def _make_padding_matrix_weights(config: Config) -> WeightTensors:
"""Like WeightTensors.make(), but scales weights by
config.is_per_out_ch_quant instead of is_per_act_token_quant.
WeightTensors.make() ties weight-scale shape to activation quant, so
fp8_tensor_token would get per-channel scales despite being a per-tensor
weight scheme. Scoped here rather than fixing WeightTensors.make(),
which other tests depend on."""
(_, w1, w1_scale, w1_gs), (_, w2, w2_scale, w2_gs) = make_test_weights(
e=config.E,
n=config.N,
k=config.K,
in_dtype=config.dtype,
quant_dtype=config.quant_dtype,
block_shape=config.quant_block_shape,
per_out_ch_quant=config.is_per_out_ch_quant,
)
return WeightTensors(
w1=w1, w2=w2, w1_scale=w1_scale, w2_scale=w2_scale, w1_gs=w1_gs, w2_gs=w2_gs
)
def _slice_unpadded_rank_tensors(
rank_tensors: RankTensors, k_unpadded: int
) -> RankTensors:
return RankTensors(
hidden_states=rank_tensors.hidden_states[:, :k_unpadded].contiguous(),
hidden_states_scale=rank_tensors.hidden_states_scale,
topk_weights=rank_tensors.topk_weights,
topk_ids=rank_tensors.topk_ids,
expert_map=rank_tensors.expert_map,
)
def _aiter_padding_matrix_worker(
pgi: ProcessGroupInfo,
vllm_config: VllmConfig,
cpu_group,
padded_config: Config,
padded_weights: WeightTensors,
verbose: bool,
hidden_pad_expected: int,
intermediate_pad_expected: int,
):
device = torch.device(f"cuda:{pgi.local_rank}")
init_workspace_manager(device)
set_random_seed(pgi.rank)
weights = copy.deepcopy(padded_weights)
weights.to_current_device()
rank_tensors = RankTensors.make(padded_config, pgi)
with mock.patch.object(
rocm_aiter_ops, "fused_moe", wraps=rocm_aiter_ops.fused_moe
) as fused_moe_mock:
mk_out = run_modular_kernel(
pgi, vllm_config, padded_config, weights, rank_tensors
)
assert fused_moe_mock.call_count > 0, (
"Expected AiterExperts.apply() to call rocm_aiter_ops.fused_moe."
)
for call in fused_moe_mock.call_args_list:
assert call.kwargs["hidden_pad"] == hidden_pad_expected, (
f"AiterExperts.apply() forwarded hidden_pad="
f"{call.kwargs['hidden_pad']}, expected {hidden_pad_expected}."
)
assert call.kwargs["intermediate_pad"] == intermediate_pad_expected, (
f"AiterExperts.apply() forwarded intermediate_pad="
f"{call.kwargs['intermediate_pad']}, expected "
f"{intermediate_pad_expected}."
)
# AiterExperts.apply() never slices its own output -- the buffer stays
# raw (padded) width; only the caller-sliced unpadded prefix is checked.
assert mk_out.shape[-1] == padded_config.K, (
f"AiterExperts output width {mk_out.shape[-1]} != raw hidden_dim "
f"{padded_config.K}."
)
mk_out = mk_out[..., :_PADDING_K_UNPADDED]
# Only the real slice is checked: beyond it is unspecified/reused memory.
assert torch.isfinite(mk_out).all(), (
"AiterExperts output must not contain NaN/Inf in the real "
"hidden_dim_unpadded-wide output slice."
)
unpadded_weights = _slice_unpadded_weights(
weights,
padded_config.quant_config,
_PADDING_K_UNPADDED,
_PADDING_N_UNPADDED,
)
unpadded_rank_tensors = _slice_unpadded_rank_tensors(
rank_tensors, _PADDING_K_UNPADDED
)
with set_current_vllm_config(vllm_config):
ref_out = reference_moe_impl(
padded_config, unpadded_weights, unpadded_rank_tensors
)
# ref_out's magnitude here (~1e-3-1e-2) is below atol=3e-2, so a zeroed-out
# mk_out would still pass check_accuracy below -- guard against that.
ref_scale = ref_out.abs().mean()
mk_scale = mk_out.abs().mean()
assert mk_scale > 0.5 * ref_scale, (
f"AiterExperts output magnitude (mean |mk_out|={mk_scale:.6f}) looks "
f"degenerate/zeroed vs. reference (mean |ref_out|={ref_scale:.6f})."
)
# Lenient check for every scheme, including unquantized: a strict
# assert_close(atol=rtol=3e-2) spuriously fails ~0.1-0.2% of elements
# on MI300 even for unquantized AiterExperts (confirmed on hardware).
check_accuracy(ref_out, mk_out, atol=3e-2, rtol=3e-2, percent=0.9)
@require_aiter_moe
@pytest.mark.parametrize("quant_config", _PADDING_QUANT_CONFIGS, ids=_PADDING_QUANT_IDS)
@pytest.mark.parametrize("mode", list(_PADDING_MODES))
def test_aiter_moe_padding_matrix(mode: str, quant_config: TestMoEQuantConfig | None):
"""See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").
Exercises AiterExperts's hidden_pad/intermediate_pad (rocm_aiter_moe.py)
across {no padding, hidden-only, intermediate-only, both} x the
unquantized + 4 fp8 quant schemes it supports. Compares against an
unpadded reference and verifies the hidden_pad/intermediate_pad
AiterExperts.apply() forwards to rocm_aiter_ops.fused_moe.
"""
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
MoEPrepareAndFinalizeNoDPEPModular,
)
pad_hidden, pad_intermediate = _PADDING_MODES[mode]
if (
quant_config is not None
and quant_config.block_shape is not None
and not pad_hidden
):
pytest.skip(
"AITER's block-quantized (128x128) CK GEMM kernel does not "
f"support the raw (unpadded) hidden_dim={_PADDING_K_UNPADDED} "
"shape on this hardware -- it raises 'wrong! device_gemm with "
"the specified compilation parameters does not support this "
"GEMM problem'. Only hidden_pad-forcing modes (raw "
f"hidden_dim={_PADDING_K_PADDED}) are exercised for this quant "
"scheme."
)
k = _PADDING_K_PADDED if pad_hidden else _PADDING_K_UNPADDED
n = _PADDING_N_PADDED if pad_intermediate else _PADDING_N_UNPADDED
config = Config(
Ms=_PADDING_M,
K=k,
N=n,
E=_PADDING_E,
topks=_PADDING_TOPK,
dtype=torch.bfloat16,
quant_config=quant_config,
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
fused_experts_type=AiterExperts,
world_size=1,
hidden_dim_unpadded=_PADDING_K_UNPADDED if pad_hidden else None,
intermediate_size_per_partition_unpadded=(
_PADDING_N_UNPADDED if pad_intermediate else None
),
)
assert config.is_valid()[0]
assert config.fe_supports_quant_scheme(), (
f"AiterExperts does not support quant scheme {quant_config}."
)
weights = _make_padding_matrix_weights(config)
vllm_config, env_dict = config.make_env_data()
hidden_pad_expected = _PADDING_HIDDEN_PAD_EXPECTED if pad_hidden else 0
intermediate_pad_expected = (
_PADDING_INTERMEDIATE_PAD_EXPECTED if pad_intermediate else 0
)
parallel_launch_with_config(
config.world_size,
_aiter_padding_matrix_worker,
vllm_config,
env_dict,
config,
weights,
False,
hidden_pad_expected,
intermediate_pad_expected,
)
@pytest.mark.skip(
reason="MXFP4 AiterExperts padding requires MI350/gfx950 hardware. "
'See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").'
)
def test_aiter_moe_padding_matrix_mxfp4():
"""Stub for the {no padding, hidden-only, intermediate-only, both} MXFP4
padding matrix once MI350/gfx950 hardware is available in CI."""
if __name__ == "__main__":
# Ability to test individual PrepareAndFinalize and FusedExperts combination
from .modular_kernel_tools.cli_args import make_config, make_config_arg_parser