diff --git a/tests/kernels/quantization/test_rocm_mxfp4.py b/tests/kernels/quantization/test_rocm_mxfp4.py index e60765eeaf..43b5daa45a 100644 --- a/tests/kernels/quantization/test_rocm_mxfp4.py +++ b/tests/kernels/quantization/test_rocm_mxfp4.py @@ -637,3 +637,64 @@ def test_aiter_fp4_gemm_skinny_shapes(M, N, K): pass_rate=pass_rate, max_violation_factor=GEMM_MAX_VIOLATION_FACTOR, ) + + +@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only") +@pytest.mark.parametrize( + ("shape", "supported"), + [ + ((2560, 80), True), # shared_expert gate/up_proj (in=2560 -> sn=80) + ((1280, 80), True), # gate/up_proj under TP=2 (out sharded) + ((2560, 8), True), # minimal aligned columns + ((32, 8), True), # minimal aligned rows and columns + ((2560, 20), False), # shared_expert down_proj (in=640 -> sn=20) + ((2560, 10), False), # down_proj under TP=2 (in 640->320 -> sn=10) + ((30, 8), False), # rows not a multiple of 32 + ], +) +def test_asm_fp4_scale_swizzle_supported_shape_rules(shape, supported): + """The ASM swizzle needs rows % 32 == 0 and columns % 8 == 0.""" + from vllm.model_executor.kernels.linear.mxfp4.aiter import ( + _asm_fp4_scale_swizzle_supported, + ) + + weight_scale = torch.empty(shape, dtype=torch.uint8, device="cpu") + assert _asm_fp4_scale_swizzle_supported(weight_scale) is supported + + +@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only") +def test_asm_fp4_scale_swizzle_rejects_non_2d(): + from vllm.model_executor.kernels.linear.mxfp4.aiter import ( + _asm_fp4_scale_swizzle_supported, + ) + + assert not _asm_fp4_scale_swizzle_supported( + torch.empty(2560, dtype=torch.uint8, device="cpu") + ) + assert not _asm_fp4_scale_swizzle_supported( + torch.empty(80, 2, 16, dtype=torch.uint8, device="cpu") + ) + + +@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only") +def test_aiter_mxfp4_process_weights_falls_back_to_triton_for_misaligned_scale(): + """A misaligned weight_scale must disable ASM and take the Triton path.""" + from torch.nn.parameter import Parameter + + from vllm.model_executor.kernels.linear.mxfp4.aiter import AiterMxfp4LinearKernel + + kernel = object.__new__(AiterMxfp4LinearKernel) + kernel.use_asm_gemm = True + kernel.out_dtype = torch.bfloat16 + + layer = torch.nn.Module() + layer.weight_scale = Parameter( + torch.zeros(2560, 10, dtype=torch.uint8, device="cpu"), requires_grad=False + ) + + kernel.process_weights_after_loading(layer) + + assert kernel.use_asm_gemm is False + # Triton path stores the transposed, contiguous scale. + assert tuple(layer.weight_scale.shape) == (10, 2560) + assert layer.weight_scale.is_contiguous() diff --git a/vllm/model_executor/kernels/linear/mxfp4/aiter.py b/vllm/model_executor/kernels/linear/mxfp4/aiter.py index d56c9b7645..13fd5db275 100644 --- a/vllm/model_executor/kernels/linear/mxfp4/aiter.py +++ b/vllm/model_executor/kernels/linear/mxfp4/aiter.py @@ -16,6 +16,22 @@ from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig logger = init_logger(__name__) + +_ASM_FP4_SCALE_ROW_MULTIPLE = 32 +_ASM_FP4_SCALE_COL_MULTIPLE = 8 + + +def _asm_fp4_scale_swizzle_supported(weight_scale: torch.Tensor) -> bool: + # The ASM swizzle reshapes weight_scale into a fixed tile layout, requiring + # scale rows divisible by 32 and scale columns divisible by 8 + if weight_scale.ndim != 2: + return False + sm, sn = weight_scale.shape + return ( + sm % _ASM_FP4_SCALE_ROW_MULTIPLE == 0 and sn % _ASM_FP4_SCALE_COL_MULTIPLE == 0 + ) + + # NOTE: Do not import aiter at module scope. Importing aiter eagerly initializes HIP # which can force the engine core to spawn instead of fork. # is_aiter_found_and_supported() checks platform + arch + library availability via @@ -169,6 +185,19 @@ class AiterMxfp4LinearKernel(MxFp4LinearKernel): return True, None def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + if self.use_asm_gemm and not _asm_fp4_scale_swizzle_supported( + layer.weight_scale.data + ): + logger.warning_once( + "AITER ASM FP4 GEMM requires weight_scale dims divisible by " + "(%d, %d), but this layer has weight_scale shape %s. Falling " + "back to the AITER Triton FP4 GEMM for this layer.", + _ASM_FP4_SCALE_ROW_MULTIPLE, + _ASM_FP4_SCALE_COL_MULTIPLE, + tuple(layer.weight_scale.data.shape), + ) + self.use_asm_gemm = False + if self.use_asm_gemm: from aiter.ops.shuffle import shuffle_weight diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index 2b16a2e494..495650eb19 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -141,17 +141,30 @@ class Qwen3NextSparseMoeBlock(nn.Module): self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe - shared_expert_group_size = get_quark_ocp_mx_group_size( - quant_config, - f"{prefix}.shared_expert.down_proj", - ) - self.replicate_shared_expert = _should_replicate_misaligned_shared_expert( - config.shared_expert_intermediate_size, - self.tp_size, - shared_expert_group_size, - parallel_config.enable_expert_parallel, - self.is_sequence_parallel, - ) + # Resolve shared-expert fusion first (when enabled, TP alignment constraint no + # longer applies) + self.is_fused_shared_expert_enabled = False + if config.shared_expert_intermediate_size > 0: + self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert( + quant_config, + prefix, + shared_expert_name="shared_expert", + ) + + if self.is_fused_shared_expert_enabled: + self.replicate_shared_expert = False + else: + shared_expert_group_size = get_quark_ocp_mx_group_size( + quant_config, + f"{prefix}.shared_expert.down_proj", + ) + self.replicate_shared_expert = _should_replicate_misaligned_shared_expert( + config.shared_expert_intermediate_size, + self.tp_size, + shared_expert_group_size, + parallel_config.enable_expert_parallel, + self.is_sequence_parallel, + ) if self.tp_size > config.num_experts: raise ValueError( f"Tensor parallel size {self.tp_size} is greater than " @@ -183,17 +196,6 @@ class Qwen3NextSparseMoeBlock(nn.Module): prefix=f"{prefix}.shared_expert_gate", ) - self.is_fused_shared_expert_enabled = False - if ( - config.shared_expert_intermediate_size > 0 - and not self.replicate_shared_expert - ): - self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert( - quant_config, - prefix, - shared_expert_name="shared_expert", - ) - if ( self.is_fused_shared_expert_enabled or config.shared_expert_intermediate_size <= 0