[ROCm] [BugFix] Fix AITER MXFP4 ASM-GEMM crash on unfused shared experts (#55213)

Signed-off-by: Colin Zeng <Colin.Zeng@amd.com>
This commit is contained in:
Colin Z
2026-09-10 05:23:36 +00:00
committed by GitHub
parent 479611c0c6
commit 588a813a60
3 changed files with 114 additions and 22 deletions
@@ -637,3 +637,64 @@ def test_aiter_fp4_gemm_skinny_shapes(M, N, K):
pass_rate=pass_rate,
max_violation_factor=GEMM_MAX_VIOLATION_FACTOR,
)
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
@pytest.mark.parametrize(
("shape", "supported"),
[
((2560, 80), True), # shared_expert gate/up_proj (in=2560 -> sn=80)
((1280, 80), True), # gate/up_proj under TP=2 (out sharded)
((2560, 8), True), # minimal aligned columns
((32, 8), True), # minimal aligned rows and columns
((2560, 20), False), # shared_expert down_proj (in=640 -> sn=20)
((2560, 10), False), # down_proj under TP=2 (in 640->320 -> sn=10)
((30, 8), False), # rows not a multiple of 32
],
)
def test_asm_fp4_scale_swizzle_supported_shape_rules(shape, supported):
"""The ASM swizzle needs rows % 32 == 0 and columns % 8 == 0."""
from vllm.model_executor.kernels.linear.mxfp4.aiter import (
_asm_fp4_scale_swizzle_supported,
)
weight_scale = torch.empty(shape, dtype=torch.uint8, device="cpu")
assert _asm_fp4_scale_swizzle_supported(weight_scale) is supported
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_asm_fp4_scale_swizzle_rejects_non_2d():
from vllm.model_executor.kernels.linear.mxfp4.aiter import (
_asm_fp4_scale_swizzle_supported,
)
assert not _asm_fp4_scale_swizzle_supported(
torch.empty(2560, dtype=torch.uint8, device="cpu")
)
assert not _asm_fp4_scale_swizzle_supported(
torch.empty(80, 2, 16, dtype=torch.uint8, device="cpu")
)
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_mxfp4_process_weights_falls_back_to_triton_for_misaligned_scale():
"""A misaligned weight_scale must disable ASM and take the Triton path."""
from torch.nn.parameter import Parameter
from vllm.model_executor.kernels.linear.mxfp4.aiter import AiterMxfp4LinearKernel
kernel = object.__new__(AiterMxfp4LinearKernel)
kernel.use_asm_gemm = True
kernel.out_dtype = torch.bfloat16
layer = torch.nn.Module()
layer.weight_scale = Parameter(
torch.zeros(2560, 10, dtype=torch.uint8, device="cpu"), requires_grad=False
)
kernel.process_weights_after_loading(layer)
assert kernel.use_asm_gemm is False
# Triton path stores the transposed, contiguous scale.
assert tuple(layer.weight_scale.shape) == (10, 2560)
assert layer.weight_scale.is_contiguous()
@@ -16,6 +16,22 @@ from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
logger = init_logger(__name__)
_ASM_FP4_SCALE_ROW_MULTIPLE = 32
_ASM_FP4_SCALE_COL_MULTIPLE = 8
def _asm_fp4_scale_swizzle_supported(weight_scale: torch.Tensor) -> bool:
# The ASM swizzle reshapes weight_scale into a fixed tile layout, requiring
# scale rows divisible by 32 and scale columns divisible by 8
if weight_scale.ndim != 2:
return False
sm, sn = weight_scale.shape
return (
sm % _ASM_FP4_SCALE_ROW_MULTIPLE == 0 and sn % _ASM_FP4_SCALE_COL_MULTIPLE == 0
)
# NOTE: Do not import aiter at module scope. Importing aiter eagerly initializes HIP
# which can force the engine core to spawn instead of fork.
# is_aiter_found_and_supported() checks platform + arch + library availability via
@@ -169,6 +185,19 @@ class AiterMxfp4LinearKernel(MxFp4LinearKernel):
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if self.use_asm_gemm and not _asm_fp4_scale_swizzle_supported(
layer.weight_scale.data
):
logger.warning_once(
"AITER ASM FP4 GEMM requires weight_scale dims divisible by "
"(%d, %d), but this layer has weight_scale shape %s. Falling "
"back to the AITER Triton FP4 GEMM for this layer.",
_ASM_FP4_SCALE_ROW_MULTIPLE,
_ASM_FP4_SCALE_COL_MULTIPLE,
tuple(layer.weight_scale.data.shape),
)
self.use_asm_gemm = False
if self.use_asm_gemm:
from aiter.ops.shuffle import shuffle_weight
+24 -22
View File
@@ -141,17 +141,30 @@ class Qwen3NextSparseMoeBlock(nn.Module):
self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe
shared_expert_group_size = get_quark_ocp_mx_group_size(
quant_config,
f"{prefix}.shared_expert.down_proj",
)
self.replicate_shared_expert = _should_replicate_misaligned_shared_expert(
config.shared_expert_intermediate_size,
self.tp_size,
shared_expert_group_size,
parallel_config.enable_expert_parallel,
self.is_sequence_parallel,
)
# Resolve shared-expert fusion first (when enabled, TP alignment constraint no
# longer applies)
self.is_fused_shared_expert_enabled = False
if config.shared_expert_intermediate_size > 0:
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config,
prefix,
shared_expert_name="shared_expert",
)
if self.is_fused_shared_expert_enabled:
self.replicate_shared_expert = False
else:
shared_expert_group_size = get_quark_ocp_mx_group_size(
quant_config,
f"{prefix}.shared_expert.down_proj",
)
self.replicate_shared_expert = _should_replicate_misaligned_shared_expert(
config.shared_expert_intermediate_size,
self.tp_size,
shared_expert_group_size,
parallel_config.enable_expert_parallel,
self.is_sequence_parallel,
)
if self.tp_size > config.num_experts:
raise ValueError(
f"Tensor parallel size {self.tp_size} is greater than "
@@ -183,17 +196,6 @@ class Qwen3NextSparseMoeBlock(nn.Module):
prefix=f"{prefix}.shared_expert_gate",
)
self.is_fused_shared_expert_enabled = False
if (
config.shared_expert_intermediate_size > 0
and not self.replicate_shared_expert
):
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config,
prefix,
shared_expert_name="shared_expert",
)
if (
self.is_fused_shared_expert_enabled
or config.shared_expert_intermediate_size <= 0