[ROCm] [BugFix] Fix AITER MXFP4 ASM-GEMM crash on unfused shared experts (#55213)
Signed-off-by: Colin Zeng <Colin.Zeng@amd.com>
This commit is contained in:
@@ -637,3 +637,64 @@ def test_aiter_fp4_gemm_skinny_shapes(M, N, K):
|
||||
pass_rate=pass_rate,
|
||||
max_violation_factor=GEMM_MAX_VIOLATION_FACTOR,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
@pytest.mark.parametrize(
|
||||
("shape", "supported"),
|
||||
[
|
||||
((2560, 80), True), # shared_expert gate/up_proj (in=2560 -> sn=80)
|
||||
((1280, 80), True), # gate/up_proj under TP=2 (out sharded)
|
||||
((2560, 8), True), # minimal aligned columns
|
||||
((32, 8), True), # minimal aligned rows and columns
|
||||
((2560, 20), False), # shared_expert down_proj (in=640 -> sn=20)
|
||||
((2560, 10), False), # down_proj under TP=2 (in 640->320 -> sn=10)
|
||||
((30, 8), False), # rows not a multiple of 32
|
||||
],
|
||||
)
|
||||
def test_asm_fp4_scale_swizzle_supported_shape_rules(shape, supported):
|
||||
"""The ASM swizzle needs rows % 32 == 0 and columns % 8 == 0."""
|
||||
from vllm.model_executor.kernels.linear.mxfp4.aiter import (
|
||||
_asm_fp4_scale_swizzle_supported,
|
||||
)
|
||||
|
||||
weight_scale = torch.empty(shape, dtype=torch.uint8, device="cpu")
|
||||
assert _asm_fp4_scale_swizzle_supported(weight_scale) is supported
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_asm_fp4_scale_swizzle_rejects_non_2d():
|
||||
from vllm.model_executor.kernels.linear.mxfp4.aiter import (
|
||||
_asm_fp4_scale_swizzle_supported,
|
||||
)
|
||||
|
||||
assert not _asm_fp4_scale_swizzle_supported(
|
||||
torch.empty(2560, dtype=torch.uint8, device="cpu")
|
||||
)
|
||||
assert not _asm_fp4_scale_swizzle_supported(
|
||||
torch.empty(80, 2, 16, dtype=torch.uint8, device="cpu")
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_mxfp4_process_weights_falls_back_to_triton_for_misaligned_scale():
|
||||
"""A misaligned weight_scale must disable ASM and take the Triton path."""
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.model_executor.kernels.linear.mxfp4.aiter import AiterMxfp4LinearKernel
|
||||
|
||||
kernel = object.__new__(AiterMxfp4LinearKernel)
|
||||
kernel.use_asm_gemm = True
|
||||
kernel.out_dtype = torch.bfloat16
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.weight_scale = Parameter(
|
||||
torch.zeros(2560, 10, dtype=torch.uint8, device="cpu"), requires_grad=False
|
||||
)
|
||||
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
assert kernel.use_asm_gemm is False
|
||||
# Triton path stores the transposed, contiguous scale.
|
||||
assert tuple(layer.weight_scale.shape) == (10, 2560)
|
||||
assert layer.weight_scale.is_contiguous()
|
||||
|
||||
@@ -16,6 +16,22 @@ from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
_ASM_FP4_SCALE_ROW_MULTIPLE = 32
|
||||
_ASM_FP4_SCALE_COL_MULTIPLE = 8
|
||||
|
||||
|
||||
def _asm_fp4_scale_swizzle_supported(weight_scale: torch.Tensor) -> bool:
|
||||
# The ASM swizzle reshapes weight_scale into a fixed tile layout, requiring
|
||||
# scale rows divisible by 32 and scale columns divisible by 8
|
||||
if weight_scale.ndim != 2:
|
||||
return False
|
||||
sm, sn = weight_scale.shape
|
||||
return (
|
||||
sm % _ASM_FP4_SCALE_ROW_MULTIPLE == 0 and sn % _ASM_FP4_SCALE_COL_MULTIPLE == 0
|
||||
)
|
||||
|
||||
|
||||
# NOTE: Do not import aiter at module scope. Importing aiter eagerly initializes HIP
|
||||
# which can force the engine core to spawn instead of fork.
|
||||
# is_aiter_found_and_supported() checks platform + arch + library availability via
|
||||
@@ -169,6 +185,19 @@ class AiterMxfp4LinearKernel(MxFp4LinearKernel):
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if self.use_asm_gemm and not _asm_fp4_scale_swizzle_supported(
|
||||
layer.weight_scale.data
|
||||
):
|
||||
logger.warning_once(
|
||||
"AITER ASM FP4 GEMM requires weight_scale dims divisible by "
|
||||
"(%d, %d), but this layer has weight_scale shape %s. Falling "
|
||||
"back to the AITER Triton FP4 GEMM for this layer.",
|
||||
_ASM_FP4_SCALE_ROW_MULTIPLE,
|
||||
_ASM_FP4_SCALE_COL_MULTIPLE,
|
||||
tuple(layer.weight_scale.data.shape),
|
||||
)
|
||||
self.use_asm_gemm = False
|
||||
|
||||
if self.use_asm_gemm:
|
||||
from aiter.ops.shuffle import shuffle_weight
|
||||
|
||||
|
||||
@@ -141,17 +141,30 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
|
||||
self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe
|
||||
|
||||
shared_expert_group_size = get_quark_ocp_mx_group_size(
|
||||
quant_config,
|
||||
f"{prefix}.shared_expert.down_proj",
|
||||
)
|
||||
self.replicate_shared_expert = _should_replicate_misaligned_shared_expert(
|
||||
config.shared_expert_intermediate_size,
|
||||
self.tp_size,
|
||||
shared_expert_group_size,
|
||||
parallel_config.enable_expert_parallel,
|
||||
self.is_sequence_parallel,
|
||||
)
|
||||
# Resolve shared-expert fusion first (when enabled, TP alignment constraint no
|
||||
# longer applies)
|
||||
self.is_fused_shared_expert_enabled = False
|
||||
if config.shared_expert_intermediate_size > 0:
|
||||
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
|
||||
quant_config,
|
||||
prefix,
|
||||
shared_expert_name="shared_expert",
|
||||
)
|
||||
|
||||
if self.is_fused_shared_expert_enabled:
|
||||
self.replicate_shared_expert = False
|
||||
else:
|
||||
shared_expert_group_size = get_quark_ocp_mx_group_size(
|
||||
quant_config,
|
||||
f"{prefix}.shared_expert.down_proj",
|
||||
)
|
||||
self.replicate_shared_expert = _should_replicate_misaligned_shared_expert(
|
||||
config.shared_expert_intermediate_size,
|
||||
self.tp_size,
|
||||
shared_expert_group_size,
|
||||
parallel_config.enable_expert_parallel,
|
||||
self.is_sequence_parallel,
|
||||
)
|
||||
if self.tp_size > config.num_experts:
|
||||
raise ValueError(
|
||||
f"Tensor parallel size {self.tp_size} is greater than "
|
||||
@@ -183,17 +196,6 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
prefix=f"{prefix}.shared_expert_gate",
|
||||
)
|
||||
|
||||
self.is_fused_shared_expert_enabled = False
|
||||
if (
|
||||
config.shared_expert_intermediate_size > 0
|
||||
and not self.replicate_shared_expert
|
||||
):
|
||||
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
|
||||
quant_config,
|
||||
prefix,
|
||||
shared_expert_name="shared_expert",
|
||||
)
|
||||
|
||||
if (
|
||||
self.is_fused_shared_expert_enabled
|
||||
or config.shared_expert_intermediate_size <= 0
|
||||
|
||||
Reference in New Issue
Block a user